{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 学习搭建神经网络"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1-Dimensions Version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "class Node:\n",
    "    def __init__(self, inputs=[]):\n",
    "        self.inputs = inputs\n",
    "        self.outputs = []\n",
    "\n",
    "        for n in self.inputs:\n",
    "            n.outputs.append(self)\n",
    "            # set 'self' node as inbound_nodes's outbound_nodes\n",
    "\n",
    "        self.value = None\n",
    "\n",
    "        self.gradients = {}\n",
    "        # keys are the inputs to this node, and their\n",
    "        # values are the partials of this node with \n",
    "        # respect to that input.\n",
    "        # \\partial{node}{input_i}\n",
    "        \n",
    "\n",
    "    def forward(self):\n",
    "        '''\n",
    "        Forward propagation. \n",
    "        Compute the output value vased on 'inbound_nodes' and store the \n",
    "        result in self.value\n",
    "        '''\n",
    "\n",
    "        raise NotImplemented\n",
    "    \n",
    "\n",
    "    def backward(self):\n",
    "\n",
    "        raise NotImplemented"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Node:\n",
    "    def __init__(self, inputs=[], name=None, is_trainable=True):\n",
    "        self.inputs = inputs\n",
    "        self.outputs = []\n",
    "        self.name = name\n",
    "        self.is_trainable = is_trainable\n",
    "        \n",
    "        for n in self.inputs:\n",
    "            n.outputs.append(self)\n",
    "        \n",
    "        self.value = None\n",
    "        \n",
    "        self.gradients = {}\n",
    "        \n",
    "    def forward(self):\n",
    "        raise NotImplementedError\n",
    "    \n",
    "    def backward(self):\n",
    "        raise NotImplementedError\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return self.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Placeholder(Node):\n",
    "    def __init__(self, name, is_trainable=True):\n",
    "        Node.__init__(self, name=name, is_trainable=is_trainable)\n",
    "        \n",
    "    def forward(self, value=None):\n",
    "        if value is not None: self.value = value\n",
    "    \n",
    "    def backward(self):\n",
    "        self.gradients = {}\n",
    "        for n in self.outputs:\n",
    "            self.gradients[self] = n.gradients[self] * 1\n",
    "\n",
    "class Linear(Node):\n",
    "    def __init__(self, x=None, weigth=None, bias=None, name=None, is_trainable=False):\n",
    "        Node.__init__(self, [x, weigth, bias], name=name, is_trainable=is_trainable)\n",
    "        \n",
    "    def forward(self):\n",
    "        k, x, b = self.inputs[1], self.inputs[0], self.inputs[2]\n",
    "        self.value = k.value * x.value + b.value\n",
    "        \n",
    "    def backward(self):\n",
    "        k, x, b = self.inputs[1], self.inputs[0], self.inputs[2]\n",
    "        \n",
    "        for n in self.outputs:\n",
    "            grad_cost = n.gradients[self]\n",
    "            \n",
    "            self.gradients[k] = grad_cost * x.value\n",
    "            \n",
    "            self.gradients[x] = grad_cost * k.value\n",
    "            \n",
    "            self.gradients[b] = grad_cost * 1\n",
    "    \n",
    "        \n",
    "class Sigmoid(Node):\n",
    "    def __init__(self, x, name=None, is_trainable=False):\n",
    "        Node.__init__(self, [x], name=name, is_trainable=is_trainable)\n",
    "        self.x = self.inputs[0]\n",
    "        \n",
    "    def _sigmoid(self, x):\n",
    "        return 1. / (1 + np.exp(-1 * x))\n",
    "    \n",
    "    def forward(self):\n",
    "        self.value = self._sigmoid(self.x.value)\n",
    "        \n",
    "    def partial(self):\n",
    "        return self._sigmoid(self.x.value) * (1 - self._sigmoid(self.x.value))\n",
    "    \n",
    "    def backward(self):\n",
    "        for n in self.outputs:\n",
    "            grad_cost = n.gradients[self]\n",
    "            self.gradients[self.x] = grad_cost * self.partial() \n",
    "    #    print(self.gradients)\n",
    "    \n",
    "    \n",
    "class Relu(Node):\n",
    "    def __init__(self, x, name=None, is_trainable=False):\n",
    "        Node.__init__(self, [x], name=name, is_trainable=is_trainable)\n",
    "        self.x = x\n",
    "        \n",
    "    def forward(self):\n",
    "        self.value = self.x.value * (self.x.value > 0)\n",
    "        \n",
    "    def backward(self):\n",
    "        for n in self.outputs:\n",
    "            grad_cost = n.gradients[self]\n",
    "            self.gradients[self.x] = grad_cost * (self.x.value > 0) \n",
    "        \n",
    "\n",
    "class L2_LOSS(Node):\n",
    "    def __init__(self, y, y_hat, name=None, is_trainable=False):\n",
    "        Node.__init__(self, [y, y_hat], name=name, is_trainable=is_trainable)\n",
    "        self.y = y\n",
    "        self.y_hat = y_hat\n",
    "        \n",
    "    def forward(self):        \n",
    "        y_v = np.array(self.y.value)\n",
    "        yhat_v = np.array(self.y_hat.value)\n",
    "        self.value = np.mean((y_v - yhat_v) ** 2)\n",
    "        \n",
    "    def backward(self):\n",
    "        # 1/n sum (y- yhat)**2\n",
    "        y_v = np.array(self.y.value)\n",
    "        yhat_v = np.array(self.y_hat.value)\n",
    "        self.gradients[self.y] = 2 * np.mean((y_v - yhat_v))\n",
    "        self.gradients[self.y_hat] = -2 * np.mean((y_v - yhat_v))\n",
    "     #   print(self.gradients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def toplogic(graph):\n",
    "    sorted_node = []\n",
    "    \n",
    "    while len(graph) > 0: \n",
    "\n",
    "        all_inputs = []\n",
    "        all_outputs = []\n",
    "        \n",
    "        for n in graph:\n",
    "            all_inputs += graph[n]\n",
    "            all_outputs.append(n)\n",
    "        \n",
    "        all_inputs = set(all_inputs)\n",
    "        all_outputs = set(all_outputs)\n",
    "    \n",
    "        need_remove = all_outputs - all_inputs  # which in all_inputs but not in all_outputs\n",
    "    \n",
    "        if len(need_remove) > 0: \n",
    "            node = random.choice(list(need_remove))\n",
    "            \n",
    "            visited_next = [node]\n",
    "            if len(graph) == 1:  visited_next += graph[node]\n",
    "                \n",
    "            graph.pop(node)\n",
    "            sorted_node += visited_next\n",
    "\n",
    "            for _, links in graph.items():\n",
    "                if node in links: links.remove(node)\n",
    "        else:\n",
    "            break\n",
    "        \n",
    "    return sorted_node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def topological_sort_feed_dict(feed_dict):\n",
    "    graph = convert_feed_dict_to_graph(feed_dict)\n",
    "    \n",
    "    return toplogic(graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_feed_dict_to_graph(feed_dict):\n",
    "    computing_graph = defaultdict(list)\n",
    "    \n",
    "    nodes = [n for n in feed_dict]\n",
    "    \n",
    "    while nodes:\n",
    "        n = nodes.pop(0) \n",
    "        \n",
    "        if isinstance(n, Placeholder):\n",
    "            n.value = feed_dict[n]\n",
    "        \n",
    "        if n in computing_graph: continue\n",
    "\n",
    "        for m in n.outputs:\n",
    "            computing_graph[n].append(m)\n",
    "            nodes.append(m)\n",
    "    \n",
    "    return computing_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(graph_order, monitor=False):\n",
    "    for node in graph_order:\n",
    "        if monitor: print('forward compuiting -- {}'.format(node))\n",
    "        node.forward()\n",
    "        \n",
    "def backward(graph_order, monitor=False):\n",
    "    for node in graph_order[::-1]:\n",
    "        if monitor: print('backward computing -- {}'.format(node))\n",
    "        node.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_one_epoch(graph_order, monitor=False):\n",
    "    forward(graph_order, monitor)\n",
    "    backward(graph_order, monitor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize(graph, learning_rate=1e-2):\n",
    "    # there are so many other update / optimization methods\n",
    "    # such as Adam, Mom, \n",
    "    for t in graph:\n",
    "        if t.is_trainable:\n",
    "            t.value += -1 * learning_rate * t.gradients[t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:41: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b60b30229894f09ba82b897ce5c401f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data = load_boston()\n",
    "X_, y_ = data['data'], data['target']\n",
    "X_rm = X_[:, 5]\n",
    "\n",
    "w1_, b1_ = np.random.normal(), np.random.normal()\n",
    "w2_, b2_ = np.random.normal(), np.random.normal()\n",
    "w3_, b3_ = np.random.normal(), np.random.normal()\n",
    "\n",
    "\n",
    "X, y = Placeholder(name='X', is_trainable=False), Placeholder(name='y', is_trainable=False)\n",
    "w1, b1 = Placeholder(name='w1'), Placeholder(name='b1')\n",
    "w2, b2 = Placeholder(name='w2'), Placeholder(name='b2')\n",
    "\n",
    "# build model\n",
    "output1 = Linear(X, w1, b1, name='linear-01')\n",
    "output2 = Sigmoid(output1, name='activation')\n",
    "#output2 = Relu(output1, name='activation')\n",
    "y_hat = Linear(output2, w2, b2, name='y_hat')\n",
    "cost = L2_LOSS(y, y_hat, name='cost')\n",
    "\n",
    "feed_dict = {\n",
    "    X: X_rm,\n",
    "    y: y_,\n",
    "    w1: w1_,\n",
    "    w2: w2_,\n",
    "    b1: b1_,\n",
    "    b2: b2_,\n",
    "}\n",
    "\n",
    "graph_sort = topological_sort_feed_dict(feed_dict)\n",
    "\n",
    "epoch = 1000\n",
    "\n",
    "batch_num = len(X_rm)\n",
    "\n",
    "learning_rate = 1e-3\n",
    "\n",
    "losses = []\n",
    "loss_history = []\n",
    "\n",
    "for e in tqdm_notebook(range(epoch)):\n",
    "    loss = 0\n",
    "    \n",
    "    for b in range(batch_num):\n",
    "        index = np.random.choice(range(len(X_rm)))\n",
    "        X.value = X_rm[index]\n",
    "        y.value = y_[index]\n",
    "    \n",
    "        run_one_epoch(graph_sort, monitor=False)\n",
    "    \n",
    "        optimize(graph_sort, learning_rate)\n",
    "        \n",
    "        loss += cost.value\n",
    "\n",
    "    # 记录loss历史\n",
    "    loss_history.append(loss)\n",
    "    losses.append(loss / batch_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fef17e75d10>]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3wUZf4H8M83lYSWAKGDAQwgRREjVRQEBPHOdnrqeWBHTzzL3al46qknKmLX+1kQ651iw4KCKL1YgNBBOoReQguQkP78/tiZ3ZnZmS3ZDcksn/frlVd2Z2d3n9nZ/c4z36eMKKVARESxJa66C0BERNHH4E5EFIMY3ImIYhCDOxFRDGJwJyKKQQnVXQAAaNSokcrMzKzuYhARucqSJUsOKKUy7B6rEcE9MzMTOTk51V0MIiJXEZFtTo8xLUNEFIMY3ImIYhCDOxFRDGJwJyKKQQzuREQxiMGdiCgGMbgTEcUgVwf3vflFePHH9dicd7y6i0JEVKO4OrjvO1qEV2dtwraDBdVdFCKiGsXVwV3H640QEZm5OriLVHcJiIhqJlcHdx1r7kREZq4O7gJW3YmI7Lg6uOtYcSciMnN1cGfOnYjInquDu04x6U5EZBIbwb26C0BEVMO4OrgzLUNEZM/VwV3HrAwRkZmrgzu7QhIR2Qsa3EWklYjMFpG1IrJGRO7RljcQkekislH7n64tFxF5VUQ2ichKEele1RvBrDsRkVkoNfcyAH9XSp0BoBeAUSLSCcBoADOVUlkAZmr3AeBiAFna30gAb0S91Brm3ImI7AUN7kqpPUqppdrtYwDWAmgB4DIAH2irfQDgcu32ZQA+VB6/AkgTkWZRL7mpjFX56kRE7hNWzl1EMgGcDWAhgCZKqT2A5wAAoLG2WgsAOwxP26kts77WSBHJEZGcvLy88EsO1tyJiJyEHNxFpA6ASQDuVUodDbSqzTK/urVSarxSKlsplZ2RkRFqMWyx4k5EZBZScBeRRHgC+0dKqS+1xfv0dIv2f7+2fCeAVoantwSwOzrFtZSLvWWIiGyF0ltGALwDYK1S6kXDQ5MB3KDdvgHAN4blI7ReM70A5Ovpm6rCnDsRkVlCCOv0BTAcwCoRWa4t+yeAsQA+E5FbAGwHcLX22FQAwwBsAlAI4KaoltiAOXciIntBg7tSagHs8+gAMNBmfQVgVITlCoti1p2IyMTlI1SJiMiOq4O7jjl3IiIzVwd35tyJiOy5OrjrWHEnIjJzeXBn1Z2IyI7Lg7sHL7NHRGTm6uDOnDsRkT13B/fqLgARUQ3l6uCuY1aGiMjM1cFdmJchIrLl6uCu4/QDRERmrg7urLcTEdlzdXDXMedORGTm6uDOlDsRkT1XB3cda+5ERGauDu68zB4RkT1XB3cdK+5ERGauDu7MuRMR2XN1cNdx4jAiIrOgwV1E3hWR/SKy2rDsUxFZrv3l6hfOFpFMETlheOzNqiw8ERHZC3qBbADvA/gPgA/1BUqpa/TbIvICgHzD+puVUt2iVcBQsN5ORGQWNLgrpeaJSKbdY+KZ3OWPAC6MbrFCw5w7EZG9SHPu/QDsU0ptNCxrIyLLRGSuiPRzeqKIjBSRHBHJycvLi6wUrLoTEZlEGtyvAzDRcH8PgNZKqbMB/A3AxyJSz+6JSqnxSqlspVR2RkZGpd6cs0ISEdmrdHAXkQQAVwL4VF+mlCpWSh3Ubi8BsBlA+0gLGQxnhSQiMouk5j4IwDql1E59gYhkiEi8drstgCwAWyIrojO93s6ekEREZqF0hZwI4BcAHURkp4jcoj10LcwpGQA4H8BKEVkB4AsAdyilDkWzwOayVdUrExG5Wyi9Za5zWH6jzbJJACZFXqzwsOJORGTm6hGqnDiMiMieq4O7jjl3IiIzVwd35tyJiOy5Orjr2BWSiMjM1cGdFXciInuuDu465tyJiMzcHdxZdScisuXu4K5hxZ2IyMzVwZ393ImI7Lk6uHsx6U5EZOLq4M5+7kRE9lwd3HWstxMRmbk6uLPiTkRkz9XBXceUOxGRmauDOy+zR0Rkz9XBXadYdSciMnF1cPdeZq9aS0FEVPO4O7gzK0NEZCuUa6i+KyL7RWS1YdnjIrJLRJZrf8MMjz0kIptEZL2IDKmqghsxK0NEZBZKzf19AENtlr+klOqm/U0FABHpBM+Fsztrz3ldROKjVVgrTj9ARGQvaHBXSs0DcCjE17sMwCdKqWKl1FYAmwD0iKB8IWHFnYjILJKc+10islJL26Rry1oA2GFYZ6e2zI+IjBSRHBHJycvLq1wJWHEnIrJV2eD+BoB2ALoB2APgBW25Xbi1rVgrpcYrpbKVUtkZGRmVLIb3tSJ6PhFRrKlUcFdK7VNKlSulKgC8DV/qZSeAVoZVWwLYHVkRnbG3DBGRvUoFdxFpZrh7BQC9J81kANeKSLKItAGQBWBRZEUkIqJwJQRbQUQmAugPoJGI7ATwGID+ItINnpRLLoDbAUAptUZEPgPwG4AyAKOUUuVVU3Sm3ImInAQN7kqp62wWvxNg/acAPBVJocLFlDsRkZnLR6iy7k5EZMfVwV2n2NOdiMjE1cGd9XYiInuuDu465tyJiMxcHdyZcicisufq4K5jxZ2IyMzVwV2fFZJpGSIiM1cHdyIisufq4K7n3NkVkojIzNXBnYiI7MVEcGfOnYjIzNXBnV0hiYjsuTq4ExGRPVcHd14gm4jInquDu46X2SMiMnN1cGfOnYjInquDu44VdyIiM1cHd1bciYjsuTq461hxJyIyCxrcReRdEdkvIqsNy54TkXUislJEvhKRNG15poicEJHl2t+bVVl4XmaPiMheKDX39wEMtSybDqCLUupMABsAPGR4bLNSqpv2d0d0ihkYc+5ERGZBg7tSah6AQ5ZlPyqlyrS7vwJoWQVlC4r1diIie9HIud8M4HvD/TYiskxE5opIP6cnichIEckRkZy8vLyICsBZIYmIzCIK7iLyMIAyAB9pi/YAaK2UOhvA3wB8LCL17J6rlBqvlMpWSmVnZGRU8v0r9TQiophX6eAuIjcA+B2A65U2RFQpVayUOqjdXgJgM4D20ShoIMy5ExGZVSq4i8hQAA8CuFQpVWhYniEi8drttgCyAGyJRkEdygGAXSGJiKwSgq0gIhMB9AfQSER2AngMnt4xyQCmawH2V61nzPkA/i0iZQDKAdyhlDpk+8JERFRlggZ3pdR1NovfcVh3EoBJkRYqbMzLEBGZuH6EKhtViYj8uT64A8y5ExFZuT64s+JOROTP9cEdYMqdiMjK9cGdk4cREflzfXAHOP0AEZGV64M76+1ERP5cH9wB5tyJiKxcH9yZcici8uf64A6wnzsRkZXrg7sw605E5Mf1wR1gzp2IyMr9wZ0VdyIiP+4P7mA/dyIiK9cHd1bciYj8uT64A2B3GSIiC9cHdxHGdiIiK/cHdyZmiIj8hBTcReRdEdkvIqsNyxqIyHQR2aj9T9eWi4i8KiKbRGSliHSvqsLrFPtCEhGZhFpzfx/AUMuy0QBmKqWyAMzU7gPAxQCytL+RAN6IvJjOOP0AEZG/kIK7UmoegEOWxZcB+EC7/QGAyw3LP1QevwJIE5Fm0Sisc/lCWy/3QAFycq2b4VNaXoFN+49FqVRERNUnkpx7E6XUHgDQ/jfWlrcAsMOw3k5tmYmIjBSRHBHJycvLq3Qhwqm4939+Dq568xfHx5+Zug6DXpyHnYcLK10eIqKaoCoaVO3irV/dWik1XimVrZTKzsjIiOgNw824F5eVo6Sswm/50u2HAQD7jhZHVB4iouoWSXDfp6dbtP/7teU7AbQyrNcSwO4I3iegUC+zV1buC+YdHpmGAc/P8VsnKcHzcRSXlUelbERE1SWS4D4ZwA3a7RsAfGNYPkLrNdMLQL6evqkqoeTcJy7eYbq/68gJ7+3PcnZg/sY8JHuDu3+tviYb+vI8dHnsh+ouBhHVIAmhrCQiEwH0B9BIRHYCeAzAWACficgtALYDuFpbfSqAYQA2ASgEcFOUy2wuW4jr5ReWOD72wBcrAQCDzmgCALYpm5ps3V42AhORWUjBXSl1ncNDA23WVQBGRVKocFknDlucewjjpq3DR7f28qZa7Lw0fQP6d/Dl+/Wau9uCOxGRletHqEKARVsP4bL/LEBRqSdX/sAXK7E49zB2BOn18srMjbji9Z+995MY3IkoRoRUc6/p1uw+CgDYcagQWU3qolRrPE2IC2+Ek1tz7kREVq6vuRvDt56cKa/w3Ap3VoJ47WBQVsHgTkTu5vrgbqQ3LJaWK+0/gzQRnZpcH9yN/dzvnrgMAFCu1bzDTa/oFX3OQ0ZEbuf64J5/otRvWVkla+57tL7vFYzuRORyMdGgalVWoQd3hYmLtmNvfhES44M3rs5e75njpsIQ2+es34/e7RoiOSG+SspKRFQVXF9zt9qTf8JbYy8tr8BDX67CKzM3hpVq2Zx3HIcKSrB6Vz5ufG8xxny3topKS0RUNWIuuC/Zdthbc79+wkLv8rKK0KP7xwu344LnZqOwxNNvfu2eo9EtJBFRFYu54F5Uap9nP15cFtbrHCsqw7PT1nlekxOJEZHLxFxwf3XmRtvl7yzYGvZrLdnmmQLY6YBBRFRTxVxw334o+hfaOFHCmjsRuUvMBPcxl3epstd2y/zuFWG0KxBRbIuZ4H5RpyZV9to1ueauDN2Aytk/n4g0MRPckxOrrh96aQ2uERt7AZXX4HIS0ckVM8E9lEFKlVWT0x3GgL5hHy/aQUQeMRPc4w3T+85/YEBUX7smpzuMUyxc+p+fOBc9EQGIoeCeEOfblBZpKVF9bWtsP3i8GDuqoFdOZejz6OiOFvnPtUNEp56YCe7GmntcmBfpCFevZ2ai37jZVfoeobKOvD1qM5EaEZ16Kj1xmIh0APCpYVFbAP8CkAbgNgB52vJ/KqWmVrqEYWhQOwk9Mhs4Pp7ZMBW5ByOvcZeW15w0jfXCInazZBLRqafSwV0ptR5ANwAQkXgAuwB8BeAmAC8ppZ6PSgnDsPTRwY6P9WrbAO/f1AMdH50W0XuoGpZ/90/LhDfNAhHFpmilZQYC2KyU2hal14u6OskJqJUY770IdmXkF5biwUkro1iqyFnnrC8Mcw4dIopN0Qru1wKYaLh/l4isFJF3RSTd7gkiMlJEckQkJy8vz26VqKid5On/rs/HHklj6+Wv/4TPcnZGpVzRoneFvPvC0z33a9iZBRFVj4iDu4gkAbgUwOfaojcAtIMnZbMHwAt2z1NKjVdKZSulsjMyMiIthqPRw84AACRrNXaJoK1164GCaBQpqvT8vz6IqwZ3ySeikygaV2K6GMBSpdQ+AND/A4CIvA3guyi8h6NPRvYKON+6Hsv1dExcJNG9BtIbVJPiPdtXkwdcEdHJE420zHUwpGREpJnhsSsArI7Cezjq1bYhburbJuh6DWonAQAGdLA/S+iX1Qhv/rl7VMt2MuhdIZMTPbuSUxAQERBhzV1EUgEMBnC7YfE4EekGQAHItTx20ky/73zUSoxHvZREHC4owVXZLQEADw7tiBG9M7391L+6sw9u+zAHf7+oA7q1SquOokZE7y3jrbkz505EiDC4K6UKATS0LBseUYmiJKtJXe/tvw7M8t5OiI9Dqwap3vtnt05HziPOXSgDUUpBwkzzVOY5gZRpvWX0mjuDOxEBMTRCNVyvX98df+rZOuT1C2y6GIabAhnx7iK0ecg3nmvXkRN4fPIab4CuDH3GyqT4eK1MlX4pIoohp2xwH9a1GZ6+oqvj49bJx3YePuG3Tt7xYnR45Hs8PnlNSO85b4O5y+foSSvx/s+5WJR7KKTn2ynXGlT13kDsCklEwCkc3IMxpm4A+8v33fJ+DorLKvD+z7kRvVck0xnoz9V7A9W0EbREVD0Y3EM047d9fsus86eXlleE1BVRT+ckaBOcVSYt81nODuw/WuRrUE1gbxki8mFwD8DY7vlpzg6/x60zMmY9/D2Gv7sw6OsWlHjy9wlaDxfrFALB7D9WhAe+WIlbP8zx9nNPjpHg/lnODgx+cW51F+Ok+2zxDmzOO17dxaAYwuAeQKIWfM87vVHIz/lp08Gg6+iNs/rVo8JNy+gBfJ9Nzb2qe8vsOFSInzcdqLLXf+CLldi4/7jrD1LhemDSSvzu1QXVXQyKIQzuASRrwf2289sGXdeYjvnvL7kBc98FxZ4LbusXGAk3kIk27lYpGGrugacf2JtfFJULefQbNxt/mhD87CRSRaU196Lk0aZ/V06cQttMVY/BPYBErTbcuXk93NYv8CjYXUd8vWke/WYNFgSo3RaXeX7E+plBSZhpGT1dpGCYWyZAWkYphV7PzMQ1b/0a1vtUB33bCktOnUBnTe8RRQODewCnZ9QB4Jmf5twAFwEBgD+/Y67N6rVzwFML/duny733i7XrnOppmVBqqWv3HMVfJy5DWXmFN4Ar5QvmenDXzyB2HCr03t6dX+R9jUicjEsL6ge8U6XmXlJWgb3a/qkpDhwvrpGT5FF4GNwt/tSzNe7s3w4AMH7EOXhr+DloWCcZdZIDD+bdZrnC04nSMmw76PmBTF21B18u2+V9TL+IdYIW3P/1zZqAwayiQuHiV+bj2xW7sfVAgaF2rryNsXrO/bVZm7A57zj6jZuNN+ZuBgAs3XYYANCoThLKKxTGfPcb9uT799sPJhqXFtx3tChgMEvUehBZa+5DX56H3s/MtH3OD2v2ouOj3+NEDavtbztYgOHvLERhifMc+/d9trzGXLJR13fsLAx4fk6lnquUwoT5W5B3rDi6haKwMbhbPH1FVzwwtCMAIC01CUM6NwUA1EtJNK3XLytwI+t9n67ABc/NAeB/XVO95m68qPcXS3biTS0YW+nrA57cut5oWqF8p/TGFM/AFzy9TZ77YT1e+HE9crVa2IHjJfh1y0FMWLAVD05aFbD8VaXn0zPRyyFIA75UmDX/vG7vMexxOCg8+/06FJVWmFJjkdi0/zjmb3S+xsCRwhL879dtQccUPD11LeZvPOA3eM1oyso9lS5nKD78JRc7D4d3xmX8voVr7Z5jGDNlLS54brZtt+CPFm7D14aKDlUdBvcQdW5eD/cP6eC9/84N54b0PKUUjlkufVdi8+N55OvVGPv9OtvXMAat85+bbcrR6n3k9bMAq9dmbTLl9D9etN1brkhU1WAp/YAXqLbrVxbtfzSui553rBiDXpyL4e8sclznH5+vxCNfr8aa3YHTXPpuCjSXUHwEhb7i9Z+QOXoKNu2370KZf6IU//pmDa57O3Bby5z1+/HqzI2VLoeuvEJh2KvzAXjOvCYs2OK3zsNfrca9hhQlVR0G9xCJCEYNON17P9TL9ZWWKxy3zEtz24c5+H7VnpD6t6/fewyDLP2+9fSDUspXc49zLo/xYLJHO1BEElQAYMuBAtuDVDh2HCrEE9+uMTUC6wO7guXcf//aAmSOnoKi0nLvmYzTXP2f5+zAj2v2hlSmc5+aEXSdQwWelEOwGq5+AAz0ScdHMIncsu1HAACDXpxrm+rSu9zuOOSrHBSXlfu1ndz43mK8OH2D7Xuc9+wsPPndb37L1+45iv/M2og56/d7l1n3WbCDn5PS8gq0eWgKPl28vVLPJw8G9ypWVlGBt+b512De/znX7+LWdnbb5Mb1g4WCZ8rfOAHiAgRrYxBaqgWEBMvB4HBBSVgNpgNfmIvHv/XNqbPtYAEyR0/BpCU7HWvdb1s+hwcnrcR7P+Xi0v94AvX+o0XYe9QTpJwOHPoBcdWufADAU1PWeoO7U5y8/4uVGPnfJQCAhVsOYrFlLp9jRaXYfrAQ6/ces3u6H19NPPD+049Z+kFHKeX32RjLHMlkoau1z8PIONldWXkFpq3ei8cn/4Z+42aH3C125+ETeGfBVtOy7QcLcfEr8/H8jxtw43uLvcutwb2ynYCOniiFUsCDk1ZVOoVz9r9/xD8+XxH283YdOWEax7H9YCEWbrEfu7Ju71G/M9jDBSURTQQYTQzuVWzjPvtT5oVbD+HAcf9Gp9cMp8dbDxTgJsOPR6f/aCsqFEorKrwjXZ3YnSEkWA4GfZ+d5dew99Wynfhu5W7H112w0fcjmKfd/vvnK3Dju/5lBoCnpq413dfPHvQaXo+nfbl4p1qxNcW192gRtK7+IQWTa8b/iqvf/AWAp51jwvwtuOL1n3H+c7Mx5OV5js/7buVuHDhejH1Hi/CbVt5gmSn9oBMfJzhUUII2D01Fp3/9gMzRUzDq46UAzPtBqcqnu+wGrxnPGN+atwV3/G8JJmppua+W7qp0o+e+Y+azhMkrdiNz9BTMWrfftLyyVwUzTn6npxHDdbiwFF8s8V3vOL+wFFe+/pPtNCJGg1+caxrHcf5zs3HNeP+01vyNeRj68nx8utg3cr2otBxnPzkdT3zrf6Zj9fWyXXjLoY0tWhjcw9TaMqFYMHZTBetmWn4MAPDC9A04UVKOnzcfwNRV9o1txpp7ebnyC9RWHy30/4HEG3L0b8/bYtuv/L5PV+Cuj5cFfG1dsaHWFuosl7WTnHsgOdXcrY3TRwpLvG0S+gyZofrH5yswZspax5y17qKX5uKuj5che8wM9Hx6prexN1jo8uXc4ZfT1htSrWdc9326HN8sd66tlpVXYNa6fX4HAf29lFJ4fc4m7M0vMu1Ta++oxyavwa0f5gQsv1MNtNSyb+6e6PmOWM9QA42WXrb9MLr9+0cctKngGM9oA321KypUyAfDZ39Yh6XbjwTd5lDHV+hdRY2pJ/1saPIKT4Xo5RkbsGLHEdvn3/vpcjzj0MYWLQzuYZp8V19Mv+/8kNefFmKu12j+xjz86e2FeO6H9baPr92jpQ+03jLBgruuh6GvvvE5xhr17PX78cvmg8gcPSXo64l48ubHikodc+Sf5+zAjkOFtqfXqcnxjq/tVHPPP1Fq6q+/OPew97axoXnwi3P9Tss/M8wPFEpQ0NfZ4HD2pb/EG3M2e3sk2T1fRLDM4UdubSf4evlu3PPJcu/zp1raZl6dtQk3v5/jV0vWDzWb845j3LT1uPOjJaaa+958/yC6O0jvokLLPu07dhbu+WQZih2CvjVQBzozmLBgK44UluLDX7Yhc/QULN3u24/G7Q10zeO2/5yK24IEa52xshBKW5f1+6FvW0WFwq0fLMbCLZ4KjPEApvdSi48TlJZX4OUZG3HZ//3kfXz+xjxM/23fSRvDweAeprTUJNNVnoL58JdtYb/HEsMX3Y7eZdIzQrXC2w0ymFsNo2y/WW6fbrnpvcUBUzFGAk/f9z5jZ2GtJV+9Oe84SsoqcP8XK3HNW7/Y9pBoVCfZ8bUda+5Fpbj4lfm2j93+3yXIP1GKvGPF2Lj/uOm0HPDMW6N7JYTeIat25WPcNOfaVUlZBd6cuxnPTltnGsS27WCBZ4ZQ7Yf/86YDjjW4QLXbORvycOdHS/HKDF9ZN2ifszVw6sc1/f/RojLTWeOMtf7piLqWsRvWmvBaQ630UIHnDOmb5btR5FC7PVxoPqtavdu/HUBXX+taPEU7O73y9Z9RVl6BF6dvMG1bsHaIGWs9B7my8oqAPayMlZnfvboABcVlyC90bnewjhof8a6n91T+iVLMWLvfW+6PFm7HE1rbk54yjBPB8SL/sgx/ZxFu+zDHdGZmnVk2miIO7iKSKyKrRGS5iORoyxqIyHQR2aj9T4+8qDXfK9d2C/s5bw0/x2/Zml2h9TJQSqGsXIXU8yUhTvx6+Iz57reIGn/0AHysqMyvv/bAF+bi+9WeZXojqZFSCuNtGpp1hwtLbNNSRwL8ILcdLESvp2eaerw45X2NuVInl/7nJ7w+xzkvuij3kLf7qqcRUCHvWDEueG4Oxnz3m7ctwKn//YodR/zaEHRl5RV45CvPteWNAUBPCVlrtBv2HcMni7Z701ab9h93fG2d9ZMpq1CmSeyMuebuT0733g72ugBQt1YCikp9U2BPmL8Foyf5Dq5z13v6/heaDkCeLpn/NvTOEUtfI6UUXpu50S+Vds8ny9HpXz84lseY/lq/7xj6Pz8HZ/37R8f1S8oqsN0wMHHN7qN4ZupanG34HHTv/ZRratA+cLwY2wydE6z7Xz8gAcBFLzm380QqomuoGgxQShknUxkNYKZSaqyIjNbuPxil96qxmtarFfZzurao77cs1IYuBc/0v9YBVnYS4+O8F9HWTViwFf3aZ/ita60t7T9aZNtVbneQYfN6esEuvv4QJF312qxNAIApd5+Hzs19n5E+6teJdfDTPIfBSNGYPVPvEgl4asr/+Hwl/qKNbv7gl23o2caTBnM6+F715s+Or718xxFvUMg3tDP48v3m8r88w/9MRM/9OvFUDnwH9/aPfI8HtQF8gTxgCNJOzju9Eb5fvRcFJWWoWysRY6aYG9P1bTPWkE+UegL94cIS7zIRX0Xg92c1R0piPF6YvgEvWL6Pek3a6RrF1tSl/hsrr1D4cc1eXNS5qWk/dX3cHPjr1kqw7fWm+91r5hk9R3201Hv7WFEpgBTv/elBGnWjparSMpcB+EC7/QGAy6vofWqUyvQdT00y5537tGuIgwWhBffCknLMXp+H7q3TQipbok3ffLveA9ZGpSe+/Q2fhFDTDccd/1safCX4Alst7QLget/uUP3o8EPadzTy4fH/+9XcUD1p6U7TgXHhVk9e1ilvHGiqZ+NBauHWQ/j9awvw7LR13obrUBr+lmwLnN5TADZZ5pB/NkAaKhxpqUkAgDv+tyTgeAhjl9zSMs/nUW5qUBWMeHcRnvl+HfqMnWVbczYqMcy9ZOS0D859agb+8tFSDHtlfsB2mMu6NQ/4vlbG2vpVb/wScN2qmlsoGsFdAfhRRJaIyEhtWROl1B4A0P43jsL71HiV6adcK9Ec3BvXTcaB4yUOa9u7uGuzoOuIwK/mDnhqiFbWU95oTkWbkujciAoA4/5wpum+HgD1HhRONXEnkQ60Ckfz+rVwv03f6sp8L6wpq1W78vHGnM1YsdNz+m+XQw+XUsDQl+3bLyKVnuo5m/xp00HMDTD9gjFld0irsRvPCEWA+RsDXz/AeBZYVFphajAdN20dyiuUYyPqoQLPe67fdwwz1/r3XosG6yBGq8cmr9er/fkAABPlSURBVK6S941GcO+rlOoO4GIAo0QkpK4kIjJSRHJEJCcvL7wfrJv9Mbul6X6ypTZdt1bgFIvdyNiWaZ5Tvh5tnGeujBOxbXhdZTP4ZeVO87JozhDYp11Dx8fiBPjjua1My0rLKkwjccO9sMnJDO5pqUneQWJGlbnwSLCAFspFYYKxuy5wtGTU9TWWHy8ObcCU3ayloXx2t2sD1ABPl1zjc16fsxkLtxwMaVK5QHMT6WcVlVUdE6lFHNyVUru1//sBfAWgB4B9ItIMALT/fodEpdR4pVS2Uio7I8M/7+tO9lW0vw1uj7FXdsXWZ4Zh3FVnmZ9hqdbVqeXcDHJz3zb4/Pbefsv1nHv2ac7t1vFxgqSE0KuQw3ud5r1tDO56jayyjD96ADjbkFKy6/deWl4R0QXEw5mjJlJO8/KfKtMXG/U1XL3s4a9Cq5na9eAKdpCzOlFa7teA/6cJC02zsjp5bPIax8dKwxxDYdX32VmOj1lHi0dLRK8qIrVFpK5+G8BFAFYDmAzgBm21GwB8E8n71FR/6O6phesByu70+9pzW+HugVm4tkdrbyC/sU+m42s2rJ3k+NiDF3dAu8Z1/JbX02r7IwNcMSpOgKT4wCkRo/TaSbiuR2u/5cbGzcqoazl43XKer3tmSpJ/+ZZuP+ytdVWmTWNGFZ1q23EK4j+sOTkNaDVJrQTfvjyZF155/+dcb3/zaIqkggEEPoOMdJ4nJ5EeMpoAWCAiKwAsAjBFKTUNwFgAg0VkI4DB2v2YM+6qM7HuyaEYPbQjWqanoGPTurjqnJZo06i2dx27OV8ev7Qzvvvred6uk/MfGAAAyGyYisaGHjdjr+xqel5SfJxtzlpvbExLTXK8YpSIIDGMmnutxDjbwVH1Q+iZE0idZPPzje0AyYn+X8e352/1zmr4r991sn3NC2x6/Dh5/6bQZvOsjJ2Hg085/M4N2d7rBVRGLZvPKJD2TfwrAydDJPPkROLLpVUznfC3QXoeRSLUQYjhiii4K6W2KKXO0v46K6We0pYfVEoNVEplaf9DG4/uMvFxglqJ8ejZtiEWPHghUpMS8PzVZ2H2P/p71zHWYIy6tKiPy7q1AAC0apCKH+49H1+P6muq2WZZfpgiYnuUN6Z2DhV48puZDc3TJMQ75NydlJYp2+6C9S1pGf3sJVQN6pjPTPqc3ggdm3oGhWU1th8c9puWi81sVNv24GL8cdS2qf0bdbHpeupk1IDKB2EnA89oErBdpUk954FdANCwduDHjdo3qRN0/hsnzeuH363XyHoWVkXxy08ojf9ntjR/BzoEGZRoPaBe3KVpwPX1702/LN93O5CaWnMnB4sfHoTrerTGvYOzQlq/Q9O6SEtNwgVZvlpow9rJaNUgxW/dcwy59dev7256rE0jT1B/6ZpuGHRGE+/yOAl9mmLA8yOx+6GkWYJr5+b1Ar6O8Ydx/5AOuM7SYFonOQHTtAPbc1d5eso4BdXUpHjbH4Kxx1GTIEHJ2oDtdDYAAMNC6IUUDr3Gbu2jbvTPYWd4b9dNTsB4yyC3R3/ne/zJyzo7vs744efgh3vPN03CpUuIE6x47CLvfbvv2A2W1OGQzk1w63n2Z4VG654cikUPD/QbfRzsSmbREij90aVFPfy5V2u/A7zdGaORtUdbliE1+uwfulpXx/1DOmL1E0Pw31t6mn6rJxuDexXJqJuMZ67s6s2Hh8qYxmlavxa+ves8v3Uu6uQL2u0ttY47LmiHyXf1xdmt0/H2iHMw9e5+ADy1+2DdEI2KSsu987s8f/VZuOocTw3dWnNOrx14+0Zrg2KyGtfBqAGnm2awzB17ifd2t1ZpaKgFhPuH+A+kaVIvGdmnpfv1V764S1PcZmhrMA4ks6sQWX+oTQ0HA2M6DQAyG5rvR+qvF3oO9IFq08bt69yiHs5saR7D0Kutr7fR8N6Ztq9x36D2GNypCUTE9r2GdW1m2o+Dz/CviaYagnHD2kl48rIu+MM5wc/SaiXGo3Fdz2d6Vitf2Yui2GvpmuxWwVeycUbTehhzeVe/LsHWA76VX08bEYz7w5mYMCIb15zr3y4F+A5mToPlLj3L128+2KyulcXgXgO1y/AElVqJ8UhLTULbDHOQMTacWk8ZE+LjvAFBxNdDJi4OYaVlfn9WM+9Q9vTURG/qI9VSA9MHqzhpoAVsa1AOtyZ3Q59MiIhfwB5zeRdTKivTEKCtPXMAcwqnZXoKhnRu6p1jxfojr20pY6i58vsGtbddrl8Q3TpY5vM7fD2gjGdLSvmnN+wCgTXY3TMoy5uq69bKfHCYfFdfPHe15wxJT939qad/gDIeJOc9MACN69VCx6Z1MaRzE9N6793o3IbxyjW+6TiCdUn97d9DTAcDO5/d3hu/PHQhnr3qTPzy0IUB17XTqK79d9F6wLeyTmKnd9kdpFWy7hvUHv8cZj+yt8ShC+W1hjPYcNtRQsXgXgN9+Ze+prz9d389D8seHey9LyLe3GywfJ3+I9ev+NMzQF/4eoYgec5pDXBQG0yVUTfZ20AmMAfBBkGCu977x/h7WvTwQPw0OrwfZ6r2A7Rub3ycoLjU9+N75BJf2iLVpmulsX3im1F9ER8nuElLN7RI86Un9APs7dqB9MGhHb1nFsHUT7E/cNntqw9v7oFzDbN1GmuJCuZBX+/ddK734KRvRu7YSzDmii6OZXnG0CjfukEqzmyZhmStHeihYWegbnICWqSloFMzc3rNmG7T97eI4K3h2ab1WqSn4L+39LB9b2Ma8HxLo/eDQzuazkAT4uLwzai+3k4GAzv6j3vs2qI+mtX37KNm9VOQO/YSnNYw9Cm4rzzb08ZlrSzFieDJy7uEPArVOt/NPYOyMPJ8+wO/U1dcY5fnUGaprAwG9xqofmqiKUWQmpSAdEsXSb22Fizto1cS9drKpzb95HUf3NwDk/7SGz9qUxoXaF/MBrWTvIE+PTUJix4e5H1O3VoJfj8WI7saeuO6tcLudaMHamutS+BrZB7SuYkpoOs1soy6yX4NrT0yG3iDtf7j6n5aOj4Z2Qsz/nYBvhrVV38DAJ48+VXntMSADsF75lhr/N6yeq/I5Lk/akA7v6CXZmywVr7afsPaSRjQobE3pXC1IUUS6IzMWCv9/p5+pseGdG6KVU8MQUpSvF8X1Sb1auGTkb3w516tA6YN6iQnoF+W/WdiDO4piXE4vXEd1K2VgGn39sMdF7Q1zYmkH7QKij0Ht4aGhvfPbu+NpY8Otu0q2yrdP7jXq5Vgam9q1SAFDWon4XQtV359z9a4d5CvLaxCKQzvdVrQ7dAHCTq1XQ3r2tRvpk2nBl7jfol0gJSTk9PKQVF33+D2uHPA6UFPKX1zivuWrXtyKDo+Os17v2PTuph2r//A4tev747Pc3aief0U7NdG2DWul4z6KYmon5KI/BOlSE1KQPfW6diSZx7FevU5LfH5kp3ewBtoXm47fdo1xM+bfaMw9UbIeEPNVSnPhcE7NK2LF/94FgZ3MqcMWqWnYO2eoygpq8CMv1/gnb88d+wlptTIkM5N8caczRh4RmN0bGquwfZu2xBvzd2C7q3TUT8lEe/d1AP9xs3C3vwix77PdmcMRnoDnl0byKVnNcfsdfvx9fLdqNAmwZowIhtnaDXpOK0xNFivIKP/3dITXy/f5XjQATwN8BPmb8Wt/dog92AB4uMEvdo2NOX47QQadGc86JSUVXgPLvpyYwDX25qOnPBUIozpvkAjr516dL01/BzMWLsPF2ltD0YignsHtcefe52G7DEzcLN25qafWF3UqQkGndHEO0HaxNt6oqRM4axW9fHS9A24qW+mbVlev95/htcxl3fB3z5d4b2Azfs3nYs2jWqbrj3gNPgtUqy5u5SIBA3sgO9HY0wpWJ/n1MDXuXl9PH5pZ8TFCR77fSecm5nuncVSr+klJcRhzOX+aYFn/3Am1o8ZigbaGcd5WY381gnk49t6IXfsJRjR2zNSVq/RJWg12fHDszHpL328AevK7i39uhiO03rfXNylKZrVT8E5p/mChPEH361VGnLHXuIX2AGgf4fGWPn4RaYgN/Nv/bHmiaHe+9YUk136ZcrdvobxEb0zcdeA03FrP/9BZyKCq7WzMr3WN6hTE1PKqH5KomNt+u0R2X7LzstqhOevPstmbZ/maSn41+87oXlaCvq0C31fBbqaVv2URG+a7JpzWyExPs4U8O/sfzrOzUw31aKHdPY07lqn6XBi/e6e3ToN/725J+LjBEM6N7WdIVLXqE4ycsdeggEdPCkgvStu/w6NTdNgdG2Rht7tGiI1KQEPX9IppN+drmV6qjdtdveFp6N/h8Y4rWFtU9tPVQV31txjXNtGtfHEpZ0DdusL1DVPd3brdHx+Rx/v/Q9u7oFvV+xGemqi7Q8oLk6QHBeP5mkpmHt/f1NwCod+WUN99syGtZOwJa8AqUnxQbuZpaUmIeeRQREPvLKmvqyn5S3SUjBhRLb3Em5dtX7U797oCbTdWqV7D3KA5+D6jyEdHN8vXau1Bptwyo717KUq5DwyCIXF5RAJ3uZza7+2tgcxwBP8jd8pAGiXUcfbi+qOC9rZNooH8tWdfcNa36hry/r4efSFaGbpThvpIKP2Tepi6zPDTL+TVumpuLFPJoZ2aRq0O3FlMbjHOBHx67MMeGotnZrXw7wNeZW6Sn27jDq416FXiNVpEXQpvLFPJuqlJHoHS+nd7OwuLm4n0NWeomlQpyZY8dhFOHqiFC3SUkzdPMOlHwjsruZTEzSqkwxYBr7Of2CAbU48EqMvDj63fCgVk3A0N1RCaiXGoai0wnaUebisFaC4OMHjlzqPU4gGBvdTVM4jg3DgeDGyx8zAhTY9E8J136D2aFA7EY9+4zz5UmUkxMfhj4aufrf2a4Mpq/YgO9M5DzvnH/3DzvFHg94WURmJ8eLN4etjB3oHmEHTasKIbL9G95OpVZgXjo+WXm0b4lfteqZ2F76JxNS7+wWdE78mk1CvHl6VsrOzVU5OaBe6pejam1+ERnWSojaQQr+wdiQ1V7fIHD0Fvds2xMSRvSJ+rcMFJSguq/AOqtp+sBCN6yWHld89FVVUKGw/VIhmabUQL1JlA4JqKhFZopTyb2gBa+6nvKYRziFyKtvy9LCovZa11t06jP7bp7K4ODENXCMfBneKqo9u7VktFyaoDtHIxRJVFQZ3iirjRRqIqPqcWgkqIqJTBIM7EVEMYnAnIopBDO5ERDGIwZ2IKAYxuBMRxSAGdyKiGMTgTkQUg2rE3DIikgdgWwQv0QjAgSgVxw1Ote0FuM2nCm5zeE5TStleQqpGBPdIiUiO0+Q5sehU216A23yq4DZHD9MyREQxiMGdiCgGxUpwH1/dBTjJTrXtBbjNpwpuc5TERM6diIjMYqXmTkREBgzuREQxyNXBXUSGish6EdkkIqOruzzRIiKtRGS2iKwVkTUico+2vIGITBeRjdr/dG25iMir2uewUkS6V+8WVI6IxIvIMhH5TrvfRkQWatv7qYgkacuTtfubtMczq7PckRCRNBH5QkTWafu79ymwn+/TvterRWSiiNSKtX0tIu+KyH4RWW1YFvZ+FZEbtPU3isgN4ZTBtcFdROIB/B+AiwF0AnCdiHSq3lJFTRmAvyulzgDQC8AobdtGA5iplMoCMFO7D3g+gyztbySAN05+kaPiHgBrDfefBfCStr2HAdyiLb8FwGGl1OkAXtLWc6tXAExTSnUEcBY82x+z+1lEWgC4G0C2UqoLgHgA1yL29vX7AIZaloW1X0WkAYDHAPQE0APAY/oBISRKKVf+AegN4AfD/YcAPFTd5aqibf0GwGAA6wE005Y1A7Beu/0WgOsM63vXc8sfgJbaF/5CAN8BEHhG7SVY9zeAHwD01m4naOtJdW9DJba5HoCt1rLH+H5uAWAHgAbavvsOwJBY3NcAMgGsrux+BXAdgLcMy03rBftzbc0dvi+Jbqe2LKZop6FnA1gIoIlSag8AaP8ba6vFwmfxMoAHAFRo9xsCOKKUKtPuG7fJu73a4/na+m7TFkAegPe0dNQEEamNGN7PSqldAJ4HsB3AHnj23RLE/r4Gwt+vEe1vNwd3u0vPx1S/ThGpA2ASgHuVUkcDrWqzzDWfhYj8DsB+pdQS42KbVVUIj7lJAoDuAN5QSp0NoAC+U3U7rt9uLa1wGYA2AJoDqA1PWsIq1vZ1IE7bGNG2uzm47wTQynC/JYDd1VSWqBORRHgC+0dKqS+1xftEpJn2eDMA+7Xlbv8s+gK4VERyAXwCT2rmZQBpIpKgrWPcJu/2ao/XB3DoZBY4SnYC2KmUWqjd/wKeYB+r+xkABgHYqpTKU0qVAvgSQB/E/r4Gwt+vEe1vNwf3xQCytFb2JHgaZSZXc5miQkQEwDsA1iqlXjQ8NBmA3mJ+Azy5eH35CK3VvReAfP30zw2UUg8ppVoqpTLh2Y+zlFLXA5gN4CptNev26p/DVdr6rqvNKaX2AtghIh20RQMB/IYY3c+a7QB6iUiq9j3Xtzmm97Um3P36A4CLRCRdO+O5SFsWmupudIiwwWIYgA0ANgN4uLrLE8XtOg+e06+VAJZrf8PgyTXOBLBR+99AW1/g6Tm0GcAqeHoiVPt2VHLb+wP4TrvdFsAiAJsAfA4gWVteS7u/SXu8bXWXO4Lt7QYgR9vXXwNIj/X9DOAJAOsArAbwXwDJsbavAUyEp02hFJ4a+C2V2a8Abta2fROAm8IpA6cfICKKQW5OyxARkQMGdyKiGMTgTkQUgxjciYhiEIM7EVEMYnAnIopBDO5ERDHo/wHoy2CssQwyVgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fef1716f090>]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD4CAYAAAAZ1BptAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXwU5f0H8M93d3ORkIQj3GAAUUAE0YigCApWEQ+s1Xq1HsWiFa/aqqg/Sz2rtVVLtbbU+wIVL5SriCKgcoT7hnCFQIBwhSOQa5/fH/PMZmZ2Zje7SUzAz/v1yivZZ2ZnZ3Y3853neb7PM6KUAhERkRtffe8AERE1XAwSRETkiUGCiIg8MUgQEZEnBgkiIvIUqO8dqG3NmzdX2dnZ9b0bRETHlIULF+5WSmU5y4+7IJGdnY3c3Nz63g0iomOKiGxxK2dzExEReWKQICIiTwwSRETkiUGCiIg8MUgQEZEnBgkiIvLEIEFERJ4YJLRPFxfg3bmuacJERD9ZDBLaxCXb8WHu1vreDSKiBoVBwoL3XyIismOQ0ESkvneBiKjBYZCwUGBVgojIikFCYz2CiCgcg4QF+ySIiOwYJDR2SRARhWOQsGBNgojIjkEiRNhtTUTkwCChsbmJiCgcg4SFYnsTEZENg4TGigQRUTgGCSIi8sQgobFPgogoHIOEBbskiIjsogYJEXldRHaJyApLWVMRmS4i6/XvJrpcRGSMiOSJyDIROd3ynJv0+utF5CZL+Rkislw/Z4zomfa8XqOuCHsliIjCVKcm8SaAIY6yUQBmKKW6AJihHwPAxQC66J8RAF4BjBM+gNEAzgLQB8Boy0n/Fb2u+bwhUV6jznCCPyIiu6hBQik1C8BeR/EwAG/pv98CcIWl/G1lmAsgU0RaA7gIwHSl1F6l1D4A0wEM0cvSlVI/KCP/9G3Httxeo06wT4KIKFy8fRItlVKFAKB/t9DlbQFYb+9WoMsilRe4lEd6jTAiMkJEckUkt6ioKM5DYp8EEZFTbXdcu12PqzjKY6KUGquUylFK5WRlZcX6dACsSRARuYk3SOzUTUXQv3fp8gIA7S3rtQOwPUp5O5fySK9RZ1iRICKyizdITARgZijdBOBzS/mNOsupL4Bi3VQ0DcCFItJEd1hfCGCaXnZQRPrqrKYbHdtye406wewmIqJwgWgriMg4AOcBaC4iBTCylJ4B8KGIDAeQD+BqvfpkAEMB5AEoAXALACil9orIEwAW6PUeV0qZneG/g5FBlQJgiv5BhNeoM5y7iYjILmqQUEpd57FosMu6CsBIj+28DuB1l/JcAD1cyve4vUadYUWCiCgMR1xbsB5BRGTHIKGxIkFEFI5BwopVCSIiGwYJTYS3LyUicmKQ0NjcREQUjkHCgimwRER2DBIap+UgIgrHIGHBegQRkR2DhMaKBBFROAYJC3ZJEBHZMUhowk4JIqIwDBIWvH0pEZEdg4TGegQRUTgGCQv2SRAR2TFImFiVICIKwyBhwZoEEZEdg4TG25cSEYVjkCAiIk8MEhqHSRARhWOQsOAssEREdgwSGisSREThGCQsWI8gIrJjkNBEmAJLROTEIKExBZaIKByDhAUn+CMismOQ0JgCS0QUjkHCgn0SRER2DBIaaxJEROEYJCxYkSAismOQCGFVgojIiUHCgn0SRER2DBIa+ySIiMIxSNiwKkFEZMUgobEiQUQUrkZBQkR+LyIrRWSFiIwTkWQR6Sgi80RkvYh8ICKJet0k/ThPL8+2bOchXb5WRC6ylA/RZXkiMqom+1od7JMgIrKLO0iISFsAdwPIUUr1AOAHcC2AZwG8oJTqAmAfgOH6KcMB7FNKnQjgBb0eRKS7ft4pAIYA+JeI+EXED+BlABcD6A7gOr1unWCfBBFRuJo2NwUApIhIAEAjAIUABgGYoJe/BeAK/fcw/Rh6+WAREV0+XilVqpTaBCAPQB/9k6eU2qiUKgMwXq9bZ1iRICKyiztIKKW2AfgbgHwYwaEYwEIA+5VSFXq1AgBt9d9tAWzVz63Q6zezljue41UeRkRGiEiuiOQWFRXFdTycBZaIKFxNmpuawLiy7wigDYBUGE1DTuYFuttZWMVRHl6o1FilVI5SKicrKyvarnvi7UuJiOxq0tx0AYBNSqkipVQ5gE8AnA0gUzc/AUA7ANv13wUA2gOAXp4BYK+13PEcr/I6wT4JIqJwNQkS+QD6ikgj3bcwGMAqAN8AuEqvcxOAz/XfE/Vj6OVfK+PSfSKAa3X2U0cAXQDMB7AAQBedLZUIo3N7Yg32NyrWI4iI7ALRV3GnlJonIhMALAJQAWAxgLEAJgEYLyJP6rLX9FNeA/COiOTBqEFcq7ezUkQ+hBFgKgCMVEpVAoCI3AlgGozMqdeVUivj3d9oBEyBJSJyijtIAIBSajSA0Y7ijTAyk5zrHgVwtcd2ngLwlEv5ZACTa7KP1SVsbyIiCsMR1xbsuCYismOQICIiTwwSFqxHEBHZMUho7JIgIgrHIGHFqgQRkQ2DhMZpOYiIwjFIWLAiQURkxyChsU+CiCgcg4QFx0kQEdkxSGisSBARhWOQsGA9gojIjkFCY58EEVE4BgkLdkkQEdkxSGicBZaIKByDhIVirwQRkQ2DhMabDhERhWOQICIiTwwSJmEKLBGRE4OExgn+iIjCMUhYsSpBRGTDIKExA5aIKByDhAVTYImI7BgkNFYkiIjCMUhYcJwEEZEdg4TGPgkionAMEhasSBAR2TFIaBwnQUQUjkHCgrcvJSKyY5DQ2CdBRBSOQcKC9QgiIjsGCY0VCSKicAwSFuySICKyY5AwsVOCiCgMgwQREXlikNDMegTTYImIqtQoSIhIpohMEJE1IrJaRPqJSFMRmS4i6/XvJnpdEZExIpInIstE5HTLdm7S668XkZss5WeIyHL9nDEiddcmxNYmIqJwNa1J/APAVKVUVwC9AKwGMArADKVUFwAz9GMAuBhAF/0zAsArACAiTQGMBnAWgD4ARpuBRa8zwvK8ITXc36hYkSAiqhJ3kBCRdAADALwGAEqpMqXUfgDDALylV3sLwBX672EA3laGuQAyRaQ1gIsATFdK7VVK7QMwHcAQvSxdKfWDMtqA3rZsq9ZxWg4ionA1qUl0AlAE4A0RWSwir4pIKoCWSqlCANC/W+j12wLYanl+gS6LVF7gUh5GREaISK6I5BYVFdXgkDigjojIqiZBIgDgdACvKKV6AziMqqYlN26X6iqO8vBCpcYqpXKUUjlZWVmR99pr51iRICIKU5MgUQCgQCk1Tz+eACNo7NRNRdC/d1nWb295fjsA26OUt3Mpr1PMbiIiqhJ3kFBK7QCwVURO1kWDAawCMBGAmaF0E4DP9d8TAdyos5z6AijWzVHTAFwoIk10h/WFAKbpZQdFpK/OarrRsq1ax4oEEVG4QA2ffxeA90QkEcBGALfACDwfishwAPkArtbrTgYwFEAegBK9LpRSe0XkCQAL9HqPK6X26r9/B+BNACkApuifOsV6BBFRlRoFCaXUEgA5LosGu6yrAIz02M7rAF53Kc8F0KMm+1hd7JMgIgrHEdcO7JIgIqrCIKHV4WBuIqJjFoOEg2KvBBFRCIMEERF5YpBwYJ8EEVEVBgmNXRJEROEYJIiIyBODhMZZYImIwjFIOLBPgoioCoOEZvZJMAWWiKgKg4TGxiYionAMEg5sbiIiqsIgoTEFlogoHIOEQ3UrEpOWFaKkrMJzecG+Ehw4Wl47O0VEVE8YJLRYUmCXbt2Pke8vwujPV3qu0//ZbzD0H7NrY9eIiOoNg4RDdW5fatYQthcfweHSSLWJI7W2X0RE9YFBQoulT2JJ/n4AwHd5e3DK6GlYsa24jvaKiKh+MUg4VKdP4u/T19keLyswgsSeQ6V48stVqKgM1sGe1b3VhQeQPWoSZq7dVd+7QkQNBINELQjqJqo/TVyJV+dswtdrjs2TbO5m49biX63eWc97QkQNBYOEg7NL4q5xi/HxwoKIz1ldeADPTFmD8gqjBhE8RgdbBPVucx4rIjIxSGjm7Ut/9+5CfLAgP1T+xdLt+MNHSyM+9715+fj3txuOm5RXjhkhIhODhMP3G/bgwY+XA6heppNVqa5JHKMVidDxMkYQkYlBQrOeGBP9xttSFmMH9JGyylrcox9fqLmJVQki0hgkXJRVBqGUivmkf1iPwD5GKxLH7H4TUd1hkNCcF88fL9qGI+WxBYmSUmP9yuCxeboNNTexIkFEGoOEh7xdh1ASY01iz+EyAMDf/7e2LnbpR8PsJiIyMUhoztPiodJyW3PTodIK3Pj6fGzdWxJ1W5v3hK+zbf+RBj8y2+xw9zFGEJHGIOFh274jtprE/1buwKx1RXjeMdo6mqVbjSk8znnma1z6zzm1uo+1zbwrH5ubiMjEIKE5M3pWFR5A8ZGqcQ/3fWiMlfDFeAYd9vJ3WL/zYM138EdwrKbuElHdYZDQnOd+n4gtSJgqgrHPy/Ts1GOjj8KMEbEGQiI6fjFIeCgsPoqHP10eVv75ku0xb8s6F1JDznwKTSfCGEFEGoOE5nZeLKuo/dlcI93Nrr6xuYmInBgkHE5rn4kzs5vU2fZjTav9MZnjJNjcREQmBgmTPjG2yUzGmdlN6+xlDkW4k119K6/k3E1EZFfjICEifhFZLCJf6scdRWSeiKwXkQ9EJFGXJ+nHeXp5tmUbD+nytSJykaV8iC7LE5FRNd3X6vD7fDhaXnc3DaqLJqzaYnbKV7LdiYi02qhJ3ANgteXxswBeUEp1AbAPwHBdPhzAPqXUiQBe0OtBRLoDuBbAKQCGAPiXDjx+AC8DuBhAdwDX6XXrhHn1HPAJjlbUXZNQeQO+a12F7lTfsOtQg+47IaIfT42ChIi0A3AJgFf1YwEwCMAEvcpbAK7Qfw/Tj6GXD9brDwMwXilVqpTaBCAPQB/9k6eU2qiUKgMwXq9bJ8xmeBGgVNck7jivM24f2LlWX8ds0mmIKvS+fbV6F+4Zv6Se94aIGoKa1iReBPAAAPPyuBmA/Uop8zK0AEBb/XdbAFsBQC8v1uuHyh3P8SqvE8kBPwDjRGnWJLq2TkdiwP0tirdv11mTWF5Q3GDSYq335l6gb2VKRD9tcQcJEbkUwC6l1EJrscuqKsqyWMvd9mWEiOSKSG5RUVGEvfaWnGAEifLKIEr17K+Jfh8S/e7RoHFSIK7XqbDUJJYXFOOyl+bgn1+vj2tbta3CEqyy0pLqcU+IqKGoSU3iHACXi8hmGE1Bg2DULDJFxDyDtgNgjj4rANAeAPTyDAB7reWO53iVh1FKjVVK5SilcrKysuI6mAQdDMorg7g6x3jZXu0zPG/Ac0Kz1Lhex1qTKCw+AgBYse1AXNuqbdYA1ijOIEhEx5e4g4RS6iGlVDulVDaMjuevlVI3APgGwFV6tZsAfK7/nqgfQy//WhmJ+RMBXKuznzoC6AJgPoAFALrobKlE/RoT493faBIC5t3oFC46pRU2P3MJWmekuGYjfTbyHLRvmhLX61iDxO5DZfqvhtHcVG6ZcoRpsEQE1M04iQcB3CcieTD6HF7T5a8BaKbL7wMwCgCUUisBfAhgFYCpAEYqpSp1v8WdAKbByJ76UK9bJ8xblpY7goJbNlKTRgn4/QUnxfU6Zsf13I17QtN+NJSMU2tN4li/FSsR1Y5aaVNQSs0EMFP/vRFGZpJznaMArvZ4/lMAnnIpnwxgcm3sYzSJAff7Wps1idM7ZGJRvjHtd6PEAOJtsq8IBlFcUo5rx86Nf2frSGVQoXNWKk5u1Rjrdh6q790hogaAI641sybhbGYxaxKX9mwTKktN8iMtOb74WlYRxLJt+21lDaQigfLKIBL8PohI1WR/RPSTxt5J7dS2GRgxoBN+3fcEW3mZboJJsKTCJgf88MV5+7aKoGqwTTkVQYWAX+AXQbCBpOUSUf1iTULz+QQPD+2G9k0b2crNDup2mSm2dU2pif6wbX15V3+MGNDJ9XXKK4O2VNOGpCKo4Pf54PcJp+YgIgCsSUR124DOOKVNBgaelIXXb86x3Yho/sODsfNAKS57aQ5aNE7C+Se3QPGRcvRom4GV293vZ11eqcI6w1UDOSFXVAaR4BP4RBDHvZWI6DjEIBGF3ycYeJIx9mJQ15a2ZS3Sk0NBo3FyAM9e1TO0TDySSMsrg7YsongppTzHcMSrotJobvIJ2CdBRADY3FRj2c1TMbhrCzz/y9Ns5V4n2b2Hy+K6BapVYfERdHxoMj5dXBAqe/O7TZi9Pr7R5qbyoNFx7fdJg5kqhIjqF4NEDSX4fXjt5jPRq32mrbwqK6o13v/tWaHyrXtLwib5m5O3Gx/lbkX2qEk4eDT8vtpOm3YfBgCMn181tdWfv1iFX782P+7jAIwUWL9P4PMxu4mIDAwSdcTMimqeloQWjasGVeTvLbFNpAcY/RT/mrkBALDrYGnUbZvzTB0tr90sqfJKhYDPZ2Q3MUYQERgk6oxZk0gM+JAUqMqAyt9bgg9yC8LWP6zvWGeO1yiNcE8Lc53auDnSpt2HMXl5IQDdca37JNjcREQAg0SdMaf3SPBL6MofAA4ercDqwvAJ/aw1iHHz83Hy/00NTQDoZLYEHamFmsSgv8/EHe8tAmCmwOrmpmM8SASDCgOf+wafL9lW37vyo9q6twTvzdtS37tBxxEGiTpi1iQS/D4kJVT/ba4IKnyUa/Q1bNvnHiTMMQy10dxk7XqoMDuu5ccZJzF5eaEtpbg2lVYEsWVPCf740dI62X5Dde3YuXjk0xUNdsAmHXsYJOpIub4ST/D7Qjc0AoDWGckRn1cZVCjR/+AfLypwHW9RqbOjartPoqJSIRCl4zoYVMjbVfN5nbbsOYw73luE+z6omzvgmTPaNtSBi3Vl96HofVpEsWCQqCNmc1Oi3xe6VwUAfHXfwIjPe23OplAz0rj5W3HJmDlh65j93ma/QW0MxlNKGR3Xfl/EwXTPTF2DC57/Flv3ltTo9cxAWOBRW6opcyzKTy1JyzxeZqdRbWGQqCNllVV9Euagt4yUBKRGuZnPuPn52LLH+wQ8aVkh/vOtkQllXiVbO5mrEzCenrwa01fttJVVBhUqg0EEfAK/r6pJ61Bphe3q9L25Rnv34bIKxCsYVFisZ9St5fGAIc4Msp+CooOloe9dQwkSS7bud51un44dDBJ15Oazs9GuSQqG9mwNAHh3+FmYfM+5rusOPbVVxG1t2n041LQ08v1FmLFmF4Cq4GBtUnn40xURt/Xmd5swdtZG/PbtXFt5RVCFRlwLjMF0ebsO4qIXZiHnya9C6x3WNYCj5UF8n7cb78yNvZP0je83h+6lES+lFJYV7PcMiuUezUwrthUje9Qk/LBhj+vyC1/4Fi81kNvJWj03bQ0+XLDVc7lSCmc+VfU5NYRpVVZuL8YVL3+HF6avi/v5Xyx1vRkl/YgYJOpIp6w0zHlwEFo0Nvog+ndpjraZ7neza5Ue+S535/9tJu58f3FYuRkcrFdq4+bn4/6Plno2B/35i1Wu5RVBFRpxvXWf8dwLnp+FbfuN5qDsUZNCaboAsDh/H65/dR4e/SxyUHJTG30aXy4rxOUvfYeJHicRr5rE/E17AQBTVxS6Ll+38xD+9r/4TmpuJiwswIEIAyS/WrUTa3ccjLqdl7/ZgAc+Xua53Nn3Uts1ibxdB0MJFdVlNiXGc28SpRQuGTMHd41b7Ppd3nu4DPd9sMT2naS6wSBRDz4feQ5+d15nAMZd7hpX494UkabccI5p+GhhAR6YEH5CcWa8WK9MKyuNmoRfT/Dnxtp/8JhHsKmOQJzTrFut14FmY9Fh1+XOUe2mRnrW3pI6zv5RSmHSskL88aOlGBXh5H7r27m46MVZNX4953cgluy0lduN2tWvX5vnuc7V//4B909YFnH8TkVlEKM+XoYNRcZnU6r75WLJ7jM9+nnVxce5f/0mbPmYGevxyeJtGB+hdkW1g0GiHvRqn4mrzmgHAMhslBi6K148znzqK9cTottJ4vZ3F9oeW69My4PGFOYJPvHsJ6jpnFMmvyVIHCmvxPb9cXRe6+Mz9/W1OZvwXd7u0GKvfU0xg4Ruvlucvw/ZoyZVq9ls3+EyPPrZimpllb39wxaMfN8Yf1JYfDTq+jXlbPePpSbx54nGXYFnr9+NO/U+O5nNjNar+m37j9jeiyVb92P8gq14UF+gmMuSA35MXbED2aMmhaU8F5eUY8763Xjok+W22t+7c/Mj7rN5IRNtPM8rMzeg12P/i7gORcYgUU/M06RSqtpX1ssK9oeVFR0sdT0hOv95KoMK367zro2Yt2kN+L2/EtWdvXbl9uKII7atx7tlTwnOfuZr2/Lb3snFBc9/G5qjyunA0XKM+ToPQNXJ4okvV+GGV+eFTvifLnYfRJekA3KJbqb4YqnR7PToZyuidvr/ddpavDN3CyYu3Y6yiiBem7Mp7OSct+sQtuw5jDmWgFUXfchHyyttJ1Xn+x1LPLc+9ctl7s1wGSkJAIyAt2r7AWwsOoRznvkaN1rmCzOb1czkjFIzSCT48G+dbLGxyN70dNGLs/Cr1+Zh3Px8LN4a/v32EtAZg9FqTM9OXYPiI+Xo/HB8d0GetKwQ2aMmYU8cqcX/W7kDxSVVQXHK8kLX5rE9h0qx64D9QqK8Mmh7bn1ikKgn5slNIfKJ2SQCXP7Sd67LPlkUfkLM3bLPdtV39b+/j7h9M+024PeeAbY6V9ArthXjkjFz8JI+iQPGyePhT5ejRGdE+f2Rg+K0lTuRt+sQzv/bTNfXnGvpdHbb0qOfrcB/vt3oum2z7d483krL2TTamApzX/wieOv7zXjiy1V454ctOFRagZHvLcKKbcW44PlvMfC5mbbsMetWt+4twYzVxrLvLYEkFuPm56Pro1Nx4iNTkD1qEnYUHw3b91hG41en1mFeRFRUKgwdMxuD/v4tAGD+5r2hGtzBo8bnm64DSqi5KeAP1ficb/EOy8nx7R+24I8fLcVl/wxP+3Ze9Jj/P9WdPibeaWbGzTdqNCu2V82S8MmiAlz/37kRByzuKD6KEe8sxF3jjb7EvF0H8bv3FuFBl6bHM578Cn2enmEru3vcYvR6PHoNqKSsAiPfXxRfbbyaGCTqiXmP7F7tMm3jKLxE+j9+btpa1/IHP16GfYfLMHt9ERblR75KK9XzQAV84nmyvGbs3IjbKNhXguv0OtapR17+Jg/vz8vHe7oJIZY+Cbd/7gRLUI01hdasDZn/4NZj9ap9mMxaQ8Avoavmg0crMHl5ISYtL8SlLic3oCotefqqnTj3r99g+Fu5uPJf3+H6V737ACJ56BN7ZtjK7cVhtbzz/zYTHy7Y6jm1CwCs2XEA+XtKPCdzXF5QHOqsNvsi3NJZb3h1Hg6VVuCA5d4qQNV7K2K5KIrwRf5i6XZMWFiA5dvCB5AecqRcm9+hyqDCrW/l4tmpazy3G41SyrPZyqxBWZvJ7vtwKb7fsAf/mpnn+hygKkhv2WPUhs2AucGjD81pyoodAIz3e9X2A3hu2hrX9+7LpYWYtKww7gyy6mCQqCfN05LwxZ398dereiLgi/4xmF+yWMzftBe3vLmgWlOIH9UngYDPF7Wd9+7BXVzLH/pkOQ7q6vTUlTtwtLwSA5/7JnRVb57Q/S7HW1pRibxd4Vk+5p7k7ynBxwsLUFYRxC1vLggtlxjvx22e5ErKKnGotAIrLCckt87+WeuKkD1qEnYeOBo6Ed8zfont8yiNctVu/m9bO7Ddgvaq7Qfw1vebI25r54Hw/g2l3PtgHvh4GW59y0h13lB0CMsL7CffIS/OxoDnvvG8ArnspTm4X78nZk1i9nr32k9JaUVo5mOzSc9sDlOqqsZXVhnE12t2ht7T6tqy257hZN5CuCKo8NXqnXhl5gbc+tYCDHvZvbYdybtzt6DTw5NdR6unpxgBz236mH0lZZ7bdJ7QzSBpvXiavmon7h4XnrVorYWWlFZixDu5ePmbDSjS+1dWEcTz09fhcGkFpugsvcbJCZ77UlO8M109OrVdBoCq9tXaVhFUrldlbo7qK+sEv3hmBplOapmG7q3TsUp/4aeuKMSQHq3DAtl/Z220DQw0O+jdahKPfLoCExYW4KXre9vKt+w5jOxmqbjyle+w+1AZurdJD3tuWQyDtcxjKymrxG3v5GJpgfv7M2tdEdo1SQl1aC/O32erdazRaasKCo9+vjLiayoovDB9HfYc9j6pLMrfhyv/ZTQJ3nR2NgBjZuADR8vRKr1qKpezHM0Sxva9m1P263btwbp5aPMzl4StU51MKHPzXh38pRXBUFDw6xOi+V7vOHAkdGW9v6Qcb+pAuGp7+ESXXuZt2hP6fwGqvkPWC4SvVhvjh6YsL0ST1MRqT1Hy0UJjVuZt+46geVoSDpVWoFGCHz6fhE6+bn0J787Nx/V9TkCT1AS0Sk+23SnSPHYJPa76jgaDCj6fhI1VuvSfszHm2t62MUSHyipCNeelW4vxs+7J+GzxNoyZsR6l5ZX4Zq3Rzzgnryi03drGmkQD4NbcdNuATjFv5/yTs8LKqtsWa9Yk/D6frZ3eTaLfZ8vIuv3dRViz44DLfTLsj80ak9/lizxFT1fuHA9i5srvPmScYJ1XdEqp0NWUl3kb94Sq/eYV95HySizYvM/zOTe+Ph+D/v5t6KqutCJou1o3p2uvztXwim0H8I8ZkQfo/cZSOzLfx1/+5wf0+8vXUT/DoFK4/yP3NNvkBB/mbXQfOBh6vuPjfvuHzci3BPcvl0Uf0Ha0vDJU2zA/X/Pzn7x8B1bqgHDHe4tCY1XcvgdezP6OA0fLcdk/54SuyN1qXmNnb8S1Y+e6ji2yWrGtGP+dZe+7OlJWiR6jp+EvU1YDgKUvxf0z+HLZdvT7y9dhqbhm85yI4FBphe1i6WBpBbJHTXLZnwO4Z/wSW4f1JwsL0DkrFYDREW5s1Pj13YaqWt26nYfwYYzjWKqLQaIBcGtuOuOEJjFto1vr9Jj+6Zy27zdOdukpAURLYkoM+EInSdOQF2dH7fgN+AVv/7AZX+sR41aHI3QCWtff57ga31B0GL//IPJMr9eMnYuBz80EUHWFt/dwmcddyN3dM36Jrd3fvOJ3DN0AABcgSURBVJI1T141td9yYjjxkSnYeeBo6MRqfjZe9h0uw/zNe12XJfh9tr4k8/2zXoE7P7U/fb7SaIbSop1sAWMEvlmT/M+sjcgeNSnq9+HG16M3gzZPS0RaUgCH9JX8lOWFWL6tONRmf9DlCj8hQvPtim3FoZPppf+cg6cmr8YyXZtUqOpL+FDf88W8V70ZI5wB26xNz9u4B3m7DoaCl/lebNp9GD1GT8NdlmYl84LFzfJtxbaa8d+nrwu9tvldM8f6rNhmr4ntiKH5LhZsbmoAUpP8YWUSx6RG1oudi3u0Cv0jVcfr320CYASn9+dFzlFP9PuQEAjfvzJHc9N+x1V/gl/wwITITTPRTF9tn3MqWmezk7V2E2s/j/W55gnQK2W0pnZYxlZYT9huRn3iPcWJsynu7vGLsXbHQdstdWtjgsijFZWhTCBTbQTQoALSkgJ4bc4m9OnYFPk6Yy8lwe+ZweXVfLtt/5FQcsGrs92z30xmbS5Uk3BkxZnMDMLPlmzHZ0uMGteUe84NJYK4OXAktvfFTJKYunIH1u086JmK/tb3m3HvBSfFtO3qYE2iATAzKGrK+tX55ZntY3ruxqLD6NkuA60zUtAyPfJ05m41CSD8/hebHRMVeo3kjoU13bd5WqLneiMGdEL/E5uHlddk4j/rlfFSlzErtclskqkp54j02et3Y9fBUlzwfNUo79qYXrxgX0lYn4szaMQjqBSU/mbf9s5CHC6tStX24pVSbp2vy22qkENHK/DUJKOZyZz7y4yf4xdsxbyNe8LSXt1uN3zxP2ZHHJkeqcPbjbVZdNa6Is/guK+OxlUwSDQAGSneJ7tILtWTB7pp0ij6Ngd3bWF73L5JIwDAE1f0sJUPOcU+AWGC3+eaoeSs+s9yDN6btrL6NZvq6NOxqeeynu0yMLhbi7DyaJ3ykViDRFE17kVeE09NXl2n27cy+3tqIlqTX7yUAnYeqHqvj4QmmPQ+CZd4zOfkbKp0+u/sjfh4kdHMZDYrmf1z2/YfwTVjw8dGeNWWIiWMxDMwz5SU4LfNLPBjYJBoAMw0u2heueF0XNyjFZaOvhCb/jLUdu9swN5skBmldvLMlafie8dMqOZ+pDmmM2+Sag84sU4jYk5BMnm5PUjUdA6nSIEwMyXRdVr2mkxbHS3VlWpfMKjwq74dQo8/0P0JkYJ97hb3hIRogdeamFAZVDhSVhlWU4jW9Gd68SvvRIWaBOXNuw/XWROnFwaJBiBT1yR+e27HUJnb6fPiU1vjlV+dgYyUBIgI7ji/M8zzbEZKAD3bZYbWTQz4kGK5t3a6ZRLB7q3TcW2fDnCeo1MTq9b5y5Wnhv52NocZQaL6V+R/vPBk1/LTY+ycd7IGyRNbpNmWZTZKCAuU2/YbqZipieF9QNWxphqztVLtOlhagd7ta/Y9qS5nE2q3P03F50tqf6ryPYfjr0m8NmdTLe5J9TBINACJAR/WPDEED13cDeefnIVHhnZDz/ZGTnjjCDcp6pyVhg1PD8UjQ7thzHW9cffgLshuZjQZJTs69h4bdkrobzMLytlHYM2xvq5P1dWbs3qc6Pd5jtJ14zWiPNKxVYc1LTHB8Q+ekZKARon27Z/zzNd447vNCPh9uLiH+z08rG9J+6aRp3C/5FTv5r661q11OsaP6PujvV7H5qk/2ms51dU4IqeaNEXGYtz8Y2vmWgaJBiJZD95545Y++O2ATmjROBmbn7kEj1zSLeLzRCS0vt8nmHLPAEy88xw0tTQRNUtNROuMqhOeGQyeu7qnbVu922fCzRDHCTUx4IspI8arIzGtGlOkR2LN3EnwC579RVXtp0mq9+y6xUfKcXVOO9dl1sPq0LSR52u3aJxkC6pZjZM8173hrA6ey+KVkRJA307NarQNa00zmjdvObNGrxWvvp2ahmX61cF4MQCIONixui7r1Sam9QecFD62yXRq26rBg/+9MSfufaopBokG7poz2+Ppn5+KMdf1jr4yjKmwzWan0Zd1B2AEhc5ZVc0x5oXZkB72K2FnMACMf8hejuCR4PfF0Njk3ffg7Puorl/mtMOE2/uF7iMOGCfia87sgJWPXYTxI/oiLSmArq0be24j0R/9BNk8zfvEH/CJLVB+fPvZnuvePrBz1NeK1V+u7BlxeY6jKe8353QM5debzJHWnZqnet4QCwCW/ulCnNDMvSYx7+HBOKllmusyk7PG9fTPT/VYs8rw/h2x5E8/w5u39AmN4DbV1RQU1ikzovnF6e3wzvA+YeVuWX+RtNaZhPdfdHJoOhMzoSQ9JYB1T16MtU8OQe8O7hdwTvFOZBgJg0QDJyK4/qwOuDzGKxQA6NfZuNJMCviQ1TgJDw/tCsB7pKvzim3W/edjwSMXoHlaEjY+PTRUnhjwhTXvRGJtLlj9+BDbduLx4JCuyMluGqpJvHBNL1xzpnG1nppUdYWdnpyARy/tHvb8SXf3d31t55QgSRH2z+cTW60jUpOIzydokxE5rTgWvzmno2vzj7Xm45ye4RdntLUdzz2Du4Ry/z+4rZ/na31yx9nIaOR9Um7ROAkfRQiQgP1zvmvQibi+GjWrtpkpyGyUiOQEP3Ky7QHPbR6leEULcF7uu/AknNslvBZQnZmSre4fcjJuPjsbw/t3xA1nnQDAMjEiBIkBH5IC/mr/r9TF/cQZJI5jZoe4+WW+SKeyDrRUcRc9+jPP53do1gjN9NW09aST4I/tpGcdARtvYLCe85J1M0kvXWPq1Nz7H93t3H1KmwzX/bi0pz0QRxrB7vdJWJ/IhNvdT7bBoMLblqvO566KXAswtfZ4j72mTfl13xPQ05wPzLHviY7aX3pKQuhklJTgw2/6G0kT1iaOS3q2xukd3DuNb+3fEV/dNxAiYktsmPGHgWHrZuogc8NZHfAHncTgdhVuTT6w9mO1TE+u9pV0ZqMEfHpH5KAFAMv/fCGW//lC/O/3A3FjvxOqtW2rdI+m0urcZdKqeVoS/nz5KUhO8GPEgE7ok90UP+/dFoC9f8yrhuK8kIlnItBo4g4SItJeRL4RkdUislJE7tHlTUVkuois17+b6HIRkTEikiciy0TkdMu2btLrrxeRmyzlZ4jIcv2cMRLPMOSfsFYZyZh0d388drnRaX1Cs1R8P2oQ7jjvxNA6TVNjH6OR6Pfh6pzqDdYb++szbAHGeuId3LWlbd1rXLbZtZXRZGTt1zCDxC3nZGPGHwaGNYdZeZ3oozULvH5zTsTBf36pChLD+3dEVuMk5GS7j9tISvDhxBZVTV/Vfe+8Xt1tQr6mqYn4Tf+OoUwy53EnBnyhpohEv892YkwO+DG8f0dsfuYSW5OUM+X3s5HnhP7OyW5iO6l3bdUYDww52dasaTLH2Vibc87tkoU/OWp57996FrrobTr7sXZZxkr0tEz0BwBzHjw/9HcwqNC7QxPbRIY36wkTTY8POwWNkxNCzVaPD+uBb+8/L2y/I/Fq8hoxoBPu+1l8o55bZSTjw9v7udbcvL6vzualhlaTqADwB6VUNwB9AYwUke4ARgGYoZTqAmCGfgwAFwPoon9GAHgFMIIKgNEAzgLQB8BoM7DodUZYnlfVVkHV4rxqbpOZ4jpTZCwdbiKCHm0z8I9rTwtb1jojGZ2yUrHmiSF44Zpe+Fn3li5bMPTv0tx29W1NuzUl6YBg/ScxT4Ai4npSsvKaFTNajWZQ15auAWbavQMAAF1apuHyXsYVn9uV6LyHB2PS3f0x7d4BaNE4vEbw5BU9QlfYXrw6/Ju6jA9Z9OjP4PdJKIA4A1yC3xdqjlr46AVI8Ptwpb5itV61W5vNnCN7zYANhPdnTb13gO3iw3R6h0ycmd0U1/XpgIeHRk7CaJGeHOpPc9aErPMSmX0aL1zTC3MfGmzrO3Jrkjevtm8b0AnjftsXN/bLDlvHa9aDKfeca3uc2SjB9j399v7zbP05nbLSPKfSt24jku6tjZmOr7dkGHp9j50d33URJOJOL1FKFQIo1H8fFJHVANoCGAbgPL3aWwBmAnhQl7+tjN6+uSKSKSKt9brTlVJ7AUBEpgMYIiIzAaQrpX7Q5W8DuALAlHj3mdytfXJIte5p4TTstLb469S12Ga5K9acBweFTq4/712VQTTy/M4Y4NKG28PSvOH2j5Cm57X6Vd8TQrfAjIVXBpCzmm7OtDnjDwNRqCfUc6tJdMpKxRu3nIkzs5siLSmAoacOtfXlnNo2AymJfrRMTw6b3iTgk1BG0a9005D1boPtmqSgwDK1iVfqcKcIgbFSp3H6fYKv7huIC543pghPDPjw5i19sGDz3tBV8F+v6ok/DzvFtv/WkfTpjqvlpIAPP+/dFtdEmfLl8WGnoHNWGsoqgji9QxP4fOJ6AeBWUTMHtDn7vLLSkkKBokfbDKx5YkioRmnlNlWLGexaZSSH+umc3LYFACe3bIxp9w5AwO9+QXJCs1R8N2oQHvl0Oea6zLb7/m/Pws4DR/H7D5bihGaNcMd5nXFpzzaYvX63Z5ZWy/Rk1yndn7uqZ+j+HoARoFo0Tka3P00NlTnnT6sNtTLBn4hkA+gNYB6AljqAQClVKCLm3AhtAVgThAt0WaTyApdyt9cfAaPGgQ4daj/d8HjnHLkdC2eGklfzzv0XdQ39/YvT2+EsPaWG82Tdu0MmFufvxwXdWuCr1buQFPBjzRNDkOj34avVxm1NY9E5Ky30D2ednjkpoep1J955TmhKks5ZaaGTgXksA07KQkqCD9NW7kSC34fzT66a7sPZAvrFXf0992XFYxfZTow922Vi5WMX4fZ3F2L2+t14+uenQgShm0S5JQc8c+WptiSGN24+09Y3Yq1J2Nv4fchIScBQS6ZRwO9DuuM1EvQx9+3UNCwLSUTwwjXhtUcntyt1N9ZgZ05zX2EJclYf3d4PQ16cFaoBOk/q/7j2NLz41Xq8e2vVxIWX92oTGjkNICy7y8r5PcxslIDh53SEzyc4uZV3lpzpKcd71TQ1EXsPl+Hszs2xWd+rvbwiGEqwcMskjObqnPaYuHQ7Zq/fje9HDUIbl4y0BlWTMIlIGoCPAdyrlDoQodvAbYGKozy8UKmxAMYCQE5Ozo8zIuYnaMo954adpOPpJfr7L3tZni/4v0u6heZhOrllYyzO3x9qNvBJ1Qnhs5HnuN78JR7NUquaKKwj1a3ME1W/Ts0wvH9HHDxas6wat6tV69QhIkD/E5uja6vGWLPjIPp1boYTmjXC4vz9+OKu/li/8xD6d7FPWni+Y/4tM2PJGV8iZWpZmcd889nZYdOx1LaBJ2Vh0t39kZ6cEBpn0rVVY0xaXoh2TewnwPZNG2Hl496tzcNOa4thp9mvIc208fw9Jdi4+zCGnOI9+NF53nrhl6eFvbexmP3A+aGA10jXhmujU/md4WdBKWXb33G/7YtZ64swvH/HqNPxxKNGQUJEEmAEiPeUUp/o4p0i0lrXIloDMG8GUADAWk9tB2C7Lj/PUT5Tl7dzWZ/qSbfW6ejW2n5nuJHnn2ibKz8et54bfoMl8+rY+s+QlhSIe2wFYNQYzPsDVOfeG9l6bECbzGQkBnyhTK/aZu2HFhFMvXcA1u08iI7NU221iWiz8wJVNQnn8VU3ZdlMZIin+TEep7Sxd0Lfcf6JGHhylmfgjkeHZo3w8e+iZzxZed1kqLqswd8c+Z9eSydwZ0Dr17mZZzNabYj7P05nGr0GYLVS6nnLookAbgLwjP79uaX8ThEZD6OTulgHkmkAnrZ0Vl8I4CGl1F4ROSgifWE0Y90I4J/x7i/Vjct6tcFlvdrg5//6Dotd7tscq7sGd8HWfSUY2qM1Zq4tChtIVRPOE0+/Ts0iTr1xXZ/2aN80xXXK8bp2UsvoTRxuzE7U03TG1zknNsN3eXuqfUOq/7u0O7Kbp2JQDa6ia8Lvk1oNEPGKlhARi7SkAP5y5an18j2qDRLvDUdEpD+A2QCWAzDrUQ/DOKF/CKADgHwAV+sTvgB4CUaGUgmAW5RSuXpbv9HPBYCnlFJv6PIcAG8CSIHRYX2XirLDOTk5Kjc3N9IqVAeOllfiUGlFxFHKsZi0rBAj31+Ei3u0wiu/OqNWttlQfZi7FQ9MWObZzhyrtTsO4qSWaRARHCmrxO5DpWgfYYoRMuw6cBQpiX4EfD6kxDkJ5LFMRBYqpcLm/6hJdtMceKdyD3ZZXwEY6bGt1wG87lKeC6BH+DOooUlO8HtmiMTDvPKNZX6hY9Uvc9rjyt5tPVNeY2XtaE1J9DNAVFOLajTn/RTx9qXUIF3QrQVuH9gZtw0I7684HtVWgCCqbQwS1CAF/D6Murhr9BWJqE7x8oWIiDwxSBARkScGCSIi8sQgQUREnhgkiIjIE4MEERF5YpAgIiJPDBJEROQp7rmbGioRKQKwJc6nNwewuxZ351jAY/5p4DH/NNTkmE9QSoXdGey4CxI1ISK5bhNcHc94zD8NPOafhro4ZjY3ERGRJwYJIiLyxCBhN7a+d6Ae8Jh/GnjMPw21fszskyAiIk+sSRARkScGCSIi8sQgoYnIEBFZKyJ5IjKqvvenNohIexH5RkRWi8hKEblHlzcVkekisl7/bqLLRUTG6PdgmYicXr9HED8R8YvIYhH5Uj/uKCLz9DF/ICKJujxJP87Ty7Prc7/jJSKZIjJBRNboz7vf8f45i8jv9fd6hYiME5Hk4+1zFpHXRWSXiKywlMX8uYrITXr99SJyUyz7wCAB44QC4GUAFwPoDuA6Eelev3tVKyoA/EEp1Q1AXwAj9XGNAjBDKdUFwAz9GDCOv4v+GQHglR9/l2vNPQBWWx4/C+AFfcz7AAzX5cMB7FNKnQjgBb3esegfAKYqpboC6AXj2I/bz1lE2gK4G0COUqoHAD+Aa3H8fc5vAhjiKIvpcxWRpgBGAzgLQB8Ao83AUi1KqZ/8D4B+AKZZHj8E4KH63q86OM7PAfwMwFoArXVZawBr9d//AXCdZf3QesfSD4B2+p9nEIAvAQiMUagB5+cNYBqAfvrvgF5P6vsYYjzedACbnPt9PH/OANoC2Aqgqf7cvgRw0fH4OQPIBrAi3s8VwHUA/mMpt60X7Yc1CYP5hTMV6LLjhq5e9wYwD0BLpVQhAOjfLfRqx8v78CKABwAE9eNmAPYrpSr0Y+txhY5ZLy/W6x9LOgEoAvCGbmJ7VURScRx/zkqpbQD+BiAfQCGMz20hju/P2RTr51qjz5tBwiAuZcdNbrCIpAH4GMC9SqkDkVZ1KTum3gcRuRTALqXUQmuxy6qqGsuOFQEApwN4RSnVG8BhVDVBuDnmj1k3lwwD0BFAGwCpMJpbnI6nzzkar2Os0bEzSBgKALS3PG4HYHs97UutEpEEGAHiPaXUJ7p4p4i01stbA9ily4+H9+EcAJeLyGYA42E0Ob0IIFNEAnod63GFjlkvzwCw98fc4VpQAKBAKTVPP54AI2gcz5/zBQA2KaWKlFLlAD4BcDaO78/ZFOvnWqPPm0HCsABAF50ZkQijA2xiPe9TjYmIAHgNwGql1POWRRMBmBkON8HoqzDLb9RZEn0BFJvV2mOFUuohpVQ7pVQ2jM/xa6XUDQC+AXCVXs15zOZ7cZVe/5i6wlRK7QCwVURO1kWDAazCcfw5w2hm6isijfT33Dzm4/Zztoj1c50G4EIRaaJrYBfqsuqp706ZhvIDYCiAdQA2AHikvvenlo6pP4xq5TIAS/TPUBhtsTMArNe/m+r1BUaW1wYAy2FkjtT7cdTg+M8D8KX+uxOA+QDyAHwEIEmXJ+vHeXp5p/re7ziP9TQAufqz/gxAk+P9cwbwGIA1AFYAeAdA0vH2OQMYB6PPpRxGjWB4PJ8rgN/oY88DcEss+8BpOYiIyBObm4iIyBODBBEReWKQICIiTwwSRETkiUGCiIg8MUgQEZEnBgkiIvL0/+0TwpKCqq1/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predicate(x, graph):\n",
    "    X.value = x\n",
    "    forward(graph)\n",
    "    return y_hat.value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22.23892266988747"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicate(7, graph_sort)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fef170e8e50>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2df5Ac5Xnnv8/M9opZEWskWPvQSEKyzyU5WIiFNegsx7ZEzjIBxBqDMMYYu6iQq+PO5ketET4XCIoUIooNTlVihzMEfDhY4tci0NnCBtkpSMFlxUrCOqRKMFgwIkY+7cpGO0izu8/9MdOrmZ737X67p3u6e+b5VKlW09Pz9tu9O99++3m/7/MQM0MQBEFIH5m4OyAIgiAEQwRcEAQhpYiAC4IgpBQRcEEQhJQiAi4IgpBSulp5sJNPPpkXLlzYykMKgiCknh07dvyOmXud21sq4AsXLsTw8HArDykIgpB6iOg3qu0SQhEEQUgpIuCCIAgpRQRcEAQhpYiAC4IgpBQRcEEQhJRi5EIhojcA/AHAJIAJZu4nojkANgFYCOANAGuZeTSabgphMTRSxMZt+3BgrIS5+RwGVy/GQF8h7m4FIuxzaeW1CfNYaez30EgR67fswVipDADosTKYYWUxOl5GhoCpao69fM7C+jWnYaCv0NSxnZ9duaQX2/ceDO363/bUHoyOlxv6HDVkko2wKuD9zPy7mm1/BeAQM28gonUAZjPzTW7t9Pf3s9gI42NopIibH38FpfLk9LaclcWdFy9NnYiHfS6tvDZhHiuN/R4aKWLwkV0oT5llQrUyhMvOno/HdhQDHVvVbyfNXP/BR3ehPFl/LlaGsPHSZaH9DohoBzP3O7c3E0K5CMCD1f8/CGCgibaEFrBx276GP+JSeRIbt+2LqUfBCftcWnltwjxWGvu9cds+Y/EGgPIU4+GX3gx8bFW/nTRz/Z3iDVT63IrvlamAM4BniGgHEV1T3fYBZn4bAKo/36/6IBFdQ0TDRDR88ODB5nssBObAWMnX9iQT9rm08tqEeaw09jtI3yY1kQKTtkyPF+b1D9qeX0wFfAUznwngPADXEtEnTQ/AzPcycz8z9/f2NqwEFVrI3HzO1/YkE/a5tPLahHmsNPY7SN+yRIHbMj1emNc/aHt+MRJwZj5Q/fkOgCcAnA3gt0R0CgBUf74TVSeFcBhcvRg5K1u3LWdlMbh6cUw9Ck7Y59LKaxPmsdLY78HVi2Fl1IKswsoQLj9nfuBjq/rtpJnrb2Ubz8XKUEu+V54uFCKaCSDDzH+o/v8zAG4HsAXAVQA2VH8+GWVHheaxJ1TawYUS9rm08tqEeaw09tve368Lpf/UOYGOreq37UIpjpWQJaqLgfs5H3vfxLpQiOiDqIy6gYrg/yMz/yURnQRgM4AFAPYDuJSZD7m1JS4UQRBqidPWmiZXls6F4jkCZ+ZfA1im2P7/AJwbTvcEQeg0nAJaHCvh5sdfAeBvFOzWvtvNwc1VkzQB1yErMQVBiIUoLZD2zaE4VgLj+M1haKQ4vU87uLJEwAVBiIUoBdTk5tAOriwRcEEQYiFKATW5ObSDK0sEXBCEWIhSQE1uDgN9Bdx58VIU8jkQgEI+l8gJTDdaWlJNEATBJkoL5ODqxUqHifPmMNBXSJVgOxEBFwQhNqIS0HZa8+CGCLggCJEQd+ritI+uTRABFwQhdKL2eAsVZBJTEITQaafUxUlGBFwQhFAZGimi2AaLZNKACLggCKFhh050pGmRTBqQGLggCKHhVv3GaeOLe5KzHRABFwQhNNxCJLWLZGSSMxwkhCIIQmjoQiSFfM44E6Bgjgi4IAihYbo8vlWZAIdGilix4TksWrcVKzY8V5eNsB2QEIogCKFhugJybj6ndKqEOcnZCWEaEXBBEELFZAWkaa6SZmiHgg1eiIALgtByWpGrpB0KNnghAi4IgidRWP6izlXSijBN3MgkpiAIrpiUJwvabpQTjO1QsMELEXBBEFzRxZLXb9kTuM2obgq1tEPBBi8khCIIgiu6mPFYqYyhkWIgQWzVBGO7p5SVEbggCK64xYxv3Lwr0Ki5EyYYW4EIuCAIrrjFjCeZA4U+WlURvt0X8oiAC4LgykBfAbN7LO37QZbAt2KCsRVx9rgRARcEwZNbLzytQXBrOTBW8jXabcUEYyfkW5FJTEEQPLGF9cbNuzDJ3PB+vsfyvWw96gnGToizywhcEAQjBvoK+PbaZcrQBzMSN9ptVZw9TkTABUEwRhf6OFwqK/ePc7SrirNbWcKRoxNtM6kpIRRBSBFJqGKjCn1s3LYvccvWnflW8j0W3n1vAmPVm007ZCeUEbggpIQkuyqSumx9oK+AF9atwusbzkdPdxfKU/Xx+7jDPM0iAi4IKSHJroo0LFtvx0lNCaEIQkpIugAlfdl6O2YnlBG4IKSETnBVRElSwzzNIAIuCCmhHQWolaQhzOMX4xAKEWUBDAMoMvMFRLQIwI8BzAHwMoArmflYNN0UBKEVVWzanaSHefziJwb+dQCvAnhf9fVdAO5m5h8T0fcBXA3geyH3TxCEGtIsQEmwQLYbRiEUIpoH4HwAP6i+JgCrADxa3eVBAANRdFAQhPSTZAtkmjGNgd8D4BsApqqvTwIwxswT1ddvAVDeSonoGiIaJqLhgwcPNtVZQRDSSZItkGnGU8CJ6AIA7zDzjtrNil0bM9wAYOZ7mbmfmft7e3sDdlMQhDSTdAtkWjGJga8AsIaI/gzACajEwO8BkCeiruoofB6AA9F1UxCENNOOHuwk4DkCZ+abmXkeMy8E8AUAzzHzFQC2A7ikuttVAJ6MrJeCIKSaMC2Q7V5lxw/N+MBvAnADEf0bKjHx+8LpkiAI7UZYHmyZDK2HWJGcPSr6+/t5eHi4ZccTBKG9WLHhOWUoppDP4YV1q2LoUWsgoh3M3O/cLrlQBKHNaSf/tUyG1iNL6QWhjWm3kIPkg6lHBFwQ2ph28l8PjRQxfmyiYXsn54OREIogpBST0IgqXgxEFHLYvRl49nbg8JsAZQGuuXFQFlj4CeDQr4HDbwG52UDpMIDJxnYoA3B1zWBuDnDeXRiaXIEjT3wd/0LPIjuj8h4D+Hc6GcUzv4GP9X1W3Z+nrgPKRxztV/vm/DlrPnDuLcDpa/2f909uAkqHKq+tmUDXDKA0CsyaF6xNQ0TABSFlDI0UcdtTezA6frwOpao82NBIEQT1CjvfIQenSNlUBRYA8NTXgHL1xsAOYeZJ4PVfHn/tbKdu36n6/Yb+K06Z+gjOpt2gmiWEBGAufoe5r9wKLJxdL5K7NwNP/JfGftT2zfnz8JuVcwDMBXf3ZuDJa4HJmjx+5SPHbxpB2vSBuFAEIUXYMW1nWMSm1o2hc2wQgLsvO8N7IvPpG4Dh+6FZZH2cbDfQfaK7KDcJM+rEu4FZ84Hrf3X89d0frYhnEJxtuWF6HD9tKhAXiiC0AaqYdi21oRFdmIRhUMT3wTX1I2Y3Jo9FKt4A1Mk7ajn8lvtrP/j5rOm+zfTHBZnEFIQU4RW7rg2NuIVJ+m5/Ru9E2b3ZXLyTwqx57q+baSuMfZvpjwsi4ELHksYl2W6i7HRjDK5eDCurHrqOjpdx3aadjUJux479kpsDWBFZ+TIWaNGn9IEcK1eZKKzl3Fsqk5N+UbXlxrm3VEJIYbbpAxFwoSNJqz9alVMEAPI5S7003SN8PTpenj7v1/7hLzD1+J+rJ/7cyHZXJjIv/JtKrBdoFE/KAos+VX2fKoIPjcBSjSzl5gADfwdctQXUf3Vju7PmV47rnCA8fS3wue9XHCEN7WfVP3VtuXH6WuCiv62eTxVrZvU1BWvTBzKJKXQkaV6SbbqyUneOKjac8CAu423uE4UqbBdKRAIlVJBJTEGoIc1Lsp21Me1FOU4RNz2XNZnnsdZEvLPdldGmiHViEAEXOpI056d2WglVHnBAf45Obu36ITJu4k3ZSjjCp3C3Uw6WpCIxcKEjCTM/ddh4Ta6aLo/XxcudzKF3te9NAYHFO41zDGlDBFzoSMLKTx02JsKnC40Ux0p1gl97jjrWZJ7XvscMvH7qFwKFTNopB0uSkRCK0LEM9BViF2wnbsJn9zXfY9Uto6/FGU4Z6Ctg+DeH8NCL+xv2va3rflyZ/bk29j3R1YMPffXvA51HmucY0oSMwAUhQZgIn5dxzDnSffilxqXeazLP48rsz/Wx72w3rIu+69lfHZL2tTWIgAtCgjARvsMl9ei7llrBn1QovufEZZNukyTPMbQTIuCCkCBMhM9kFFu7T1YRI3GbuMSs+U1bBZM6x9BuSAxcEBKE0+Otst8Nrl7smpHQKfiXnzNfGQNXwQAopGXfSZxjaDdEwAUhYXgJX63IF8dKIDoeF5/dY+HWC0+r+/wdA0vxxMtFHDlWEfzbuu7Xtj2R7YFVM/oWL3eyEQEXhATiJZy2u+RHL+6vm9R8rzylaA0Yr4r3mszz+LLGecIM7Fy2Hh+rvv7W0CuV9quvdQuGhPgQAReEhGGy0nJopFgnrja1DpT1W/ZgrDrhmamO0tdbP9TaBhnAF1+cj43zKz5yt/ZFwJOBCLggJAwTL/jGbfu0iQaLYyUMPrIL5anje9j/nQ395OUBPhnlKZ6+AejaFy93chABF4SICBo/NvGCu4lolqhOvE1gBv5qYq1n24B4uZOECLggRIBpwqna/W2xzxApvdvOaju6epeqz9oc4hNxksJCeAQzsGXqE3XH0bUvXu7kID5wQYgAP7lAnPlPVAJMAFYu6Z1+rfKLE4CPf2iOtnyknffE2fxRzuKb5asBAFaGMLh6sbb9K5YvkPh3gpARuCBEgJ9cIF6FioFKPPqxHUX0nzqnzmboDNHoYuNrMs/jr6170U0Tx9tkYBQnYn35y9gy9QnkcxbWr6m3IIqFMNmIgAtCBPjJN246KeicyFT5xa/ftFP52fXWD+vEG0DFjcLAz7Kfwj2XyirJNCICLggRsHJJb4MNT5cLxLTwAmA2wahqS+c+mU3v1i1xt2PxxbESqvoOQDzgSUVi4IIQMkMjRTy2o1gn3gTg82epV1gOrl6sjVs7mZWzXN83LeJQ269a8bZj8UCjjVDyeScPEXBBCBlVTJsBbN97ULn/QF8BVyxfYNT2kWMTrlVtnEmkbI7qHrar1dSHRoq4cfMuz1i8eMCThQi4IISMacWcWu4YWIrZPe6jawAoT3LdKFhVfm2gr4AX1q3C6xvORyGfw21d92MGJhStEXDeXdMjbzf7oY14wJOFCLgghIybyKlKpNkirKuy48S+QZiUXxtcvRhf0lbdYeD0tUYuGEDyeScREXBBCBmvOHRtLNkZdzbBvkG4ec3tm8L1m3a6F26Ae1jE/qjk804mni4UIjoBwD8BmFHd/1FmvpWIFgH4MYA5AF4GcCUzH4uys4KQBpzpXlXY29dv2WM0+rWpHQW7hWrq8oUz4DZLqnOuZInw7bXLRLQTjMkI/CiAVcy8DMAZAD5LRMsB3AXgbmb+MIBRAFdH101BaC8IlXStYy7l0VSae+aCWdOCqnOkZInqbgraCczumQD0VYBEvJOP5wicmRmYNpFa1X8MYBWAL1a3PwhgPYDvhd9FQYiGqIoVOPOgqGCoiw3bZDX5UP75tUP41tAreHrX20rxtzL1iazWZJ5HF1Q5wgm44B4AZlWAhGRitJCHiLIAdgD4jwD+FsBrAMaY2Z7afguA8rdNRNcAuAYAFiwws0oJQtT4TTblB9NJQTfXh+49hjpPt82JJ3Shp7vreIjG+iG6SCHg3T11dS+l/Fk6MZrEZOZJZj4DwDwAZwP4iGo3zWfvZeZ+Zu7v7e1V7SIILcdPsikTau18phOSqmLDQKUsWsHFyeJm9hsdL2P82ASs6sylNv/3sSNGfRSSjS8XCjOPAfgFgOUA8kRkj+DnATgQbtcEITr8JJvywmnnM8HKEpZ/cLbyvfNPP8XX6kwno+NlgIANJzwYsAUhLXgKOBH1ElG++v8cgD8F8CqA7QAuqe52FYAno+qkIISNzqvtZ6GKPeq+btNOX04SoLIg54XXDinf27734PTqTKeIm4p6eZJxCf9MWz7NXoEppBuTEfgpALYT0W4A/wLgZ8z8NICbANxARP8G4CQA90XXTUEIF53zwnShShD/tin2U8AdA0txxfIF06GWLBE+/qE5xrlOssrJyyrn3dV0P4X4MXGh7AbQp9j+a1Ti4YKQOoI6L2qz9UWF/RTgrAo/yYyX9x/G588qYPveg9P9Hj820bCK0y7eoIQydROYQnqRdLJCrERl5TPBr/PCxB5oY2UJM7u7XH3eus8Nrl7sWnV++96DeGHdKtd+faNrsz58ctZXffVJSC4i4EJsRGnliwJTe2DBcSNauG6r+UH4+LFMq8Lbx7lx865p++Fc+p3+GBd8x7w/QqJJlYDHOVoTwsfNypfE36uXQyVnZZX5QnSLclSUp3j6b1xHXpG10D6mfUM8wCdjnkrEZ8036oeQDlKTzMok85qQLsK08rUCN4eKW7Kny8/xJ5rFsZJr4YZ331PnBK/NBb5xYi1KmFG/g5UDzr3FV1+EZJOaEXjaRmuCN37qRkaF6qkOUE9uDq5e3BBr1o26ne1aGaDsYgqpJUukj1/j+Chd9Xd/PK6/CtjdBzx7O3D4LWDWvIp4y+RlW5EaAU/baE3wRieIrco5rYrBDz6yC6CKj9re5ozLe4XxVO1aWaqKuHcoZZIZYx65wT3/7ndvFvHuAFIj4EkYrQnhEncSJdVTnUpga5/0TJwrynYnGTO7s5gqT2GSGVkizOgijGuG5V4yz6hMjuZzFtavOa2+T7s3A09eC0xWszsffrPyGhARbzNSEwNvduGFIDjx8/Rmsq+9MlPnET9ybHJ6MnOSGeVJhpUNumC+wlipjMFHdtXHxH9y03Hxtpk8VtkutBWpEXBnsVapEJJ+4p6Y9vP05rVvkJWZ5SlGV4a0Sa1qcdvHjolPU1Iv0dduF1JLakIogKS8bDfinphWxeCtDNXFwAGzJz1Tj7iTksHM5hsbzgcALFq31dgbLnQGqRJwob2Ie2JaF4NXbfO6obj1uZDP4cjRCd+rMoH6kbduHsh+D0Al/q1DEli1HSLgQmwkYWJa91Tn9wlAdy6FfA4vrFvlaxl+LbUe8sHVizH46K66pwOg8tQw/YTgFueWBFZthwi4EBtx2wiBRr/2yiW9dYmiTF0xXueiGu2rklDZZIlw+TnzccfA0ultdhu3PbVn+nN1LpTdm93j3OJAaTtEwIXYiMNGWCvY+R4L7743MW0dLI6V8NCL+6f31eVmGRopKkX0zouX1m2f0eXuETj/9FPw2I5iw6hcaQ2s4jYPNP6TW9CjO5gsoW9LiA1zNIRBf38/Dw8Pt+x4Qmeiy5kTNIyRJcIUM+bmc1h4Uk5biKHHyqA8yXVeckLFsz3bcbMAKiP0z59VwNbdbzeMxO0VnoDZDW5opIg1Q6chQ5rv88X/U0bgKYaIdjBzv3O7jMCFtqA2T7ctmkD9KDqoU8T2bhfHSq42QdWiHLsfqlCJnRq2p7ur4f1SeRLrt+zB0Ykpo2yNG7ftw5/wTJxEihqYuTki3m1KanzggqDD6cFW5dC++fHdkRZhCEpxrKR1sIyVysaFl/t//zP8Eb3XsP0oZ2Xyso0RARdSj8nI2sRvHQdZIt+uG5Xg39z9CLppomF7iXpk9N3GSAhFSC2tKG8WNZPMGD82AStDDfHxE6yMMvQyN59riPM/D3UBh1lQhFSEtkEEXAidVhTeCDohmUScIm27UAAorYkrl/Q2ZDscnTETcxTxb5o1L8KeC3EjIRQhVFqV3yTohGQaODpRCffo8v9s33uw7tzXZJ7HTCieQrLdUsChzem4EbiUZYsW0/wmblY/k99PO+f+8Epfe/2mnXWvv9G1GTNIcTPrPlHi321ORwl42orophGT/Ca638Pwbw7VLWxx+/245QVpB9xuUM5zL+gKGJdGw+6WkDA6KoTiNjoUwkHnqKjdvn7LHuXv4eGX3jT+/axc0htCb5OLmzOlNjf+bV336xuR+Hfb01Ej8Liz33UCXjlBhkaK2qx8usrtB8ZKGBopYv2WPYEy+qUNr3ww9tPIS09+H1/mn+vrZ0r8u+3pqBG4yehQaA6vwhtuTzu6ogX5HguDj+zqCPHOEhkVKhnoK+DO3EOuxY8l/t3+dNQIPAnZ7zoBt4RLbk87l58zvyG5U87KgtmsGHA7MMmV6jrXb9rpPbkrFXY6no4S8LiL6MZNEhw4usnH2T0W7hhYiv5T50wvzskSoVSebFu7oAoCpq+P1+TuRdnK/kqkeENstPJ71lECDnRuWbakOHB0T0G3XnhaXV/iXKSTJdLG46NspzYJl409uets5+9xu3tjkv8kFlr9PeuoGHgnkxQHjklx6rgX6YQh3oV8zrWdQj6HLy1fUHcddHs727mt6378SWaPfvRtzZT4d0y0+nvWcSPwTiVJDhyvp6C0u4IIlScNXZ4Wu8yakxUbnlPu7xzJX5l1cZ4AwIX3BOm2EAKt/p7JCLxDiNqBMzRSxIoNz2HRuq1YseE530vnaz+fcVWn5HPF8gUY6CvU+bVt7Fwmqmul2//yc+bX+b49r46MvmOj1U43GYF3CFE6cJqN+zk/H0YIIy7yOWu6jqVq0nzlkl7P1aaqCbD+U+dg59Z78eWyx+ibZEwWJ612uklJtQ4iqtlx3aO/LlRg+nm7lFm+xwIzcLhUTvQSersMmts1bepa3bXIvWgxAPRfDVzwHZPuChERxfdMSqoJkTlwdPG94lgJKzY8Fzgx1RQzXt9wPoD6L0XcEAFzZ+WmrY6TzChUzw+A6zkHjpF6VZwHgJOXiHgngFY63TwFnIjmA/ghgP8AYArAvcz8XSKaA2ATgIUA3gCwlpkle04Hku+xlIUHVJ5mwDwxVYYIi9ZtRU93FkeOJccLzgzlaNkklKQ7V22M9OkbgOH70WgwVPDfXjI7AaFtMAmYTQC4kZk/AmA5gGuJ6I8BrAPwLDN/GMCz1ddChzE0UsS77zWW8gLUnmaVnUo1eQdUYuEMJEq8gUq4w8nQSBE3bt7laSHTTVQqY6R3LgCG74ORePdfbdR3ob3wHIEz89sA3q7+/w9E9CqAAoCLAHy6utuDAH4B4KZIeikklo3b9vla5q4KFTgn7zIhLaSJioUn1Qu4PfJ2S8Zl47ka+ME1wOu/9NehRZ+S0EmH4isGTkQLAfQBeAnAB6riDmZ+m4jer/nMNQCuAYAFCxY001chgfiNSTtDBc4JnyuWL8BDL+4Ps4uh88Jrh/CtoVem3SZeC4+c5zzQV8BA8duV0fV7AJ6s/vMLZYHPfV9sgx2MsQuFiE4E8EsAf8nMjxPRGDPna94fZebZbm007ULZvRl49nbg8FuVXMfn3uL/j7fZNp6+AdjxAMCTlS/QWV/xHv2YHFO3z+7NwE9uOj6BlZtTWSbt1efp9t40O6/umcDpXwD+9ZlKH3LVX2XpUOU8eRKYNb/Sr/0vTl8DBoye8AGgDEIm04UursTL/Xw2idh2Ps+vELnkLAl+dODie0W8OwSdC8VIwInIAvA0gG3M/J3qtn0APl0dfZ8C4BfM7Gp2bErAd28GnvoaUK4Z8Vk54MK/Mf8jbraNp2+oxiQduFm3TI6p22fZF4GR/wVMHqtvM2MBA3+n77OqvdDIoDKXLcSK2AU7Cp2Ae05iEhEBuA/Aq7Z4V9kC4Krq/69CsIdAc569vVGQyqXK9la1seMBf9tNj6nbZ8cDjeINAFNl9z6r2gsNEe/YkZi3UMUkBr4CwJUAXiEiu5rqNwFsALCZiK4GsB/ApdF0scrht/xtj6IN1sQ5ddtNj6nbJ0i7Xu8J6UZG3kINJi6U56EP4Z0bbndcmDVPHc/1U/ev2TbsWLBqezPH1O2jO57z86bHFNLLok8BV22JuxdCwkhP4oRzb6nEhWuxcv7q/jXbxllf8bfd9Ji6fc76CibJamwzY7n3WdVeaKTnTyb1nLwEWH+48k/EW1CQnqX09oRdMw6SZtuwH139uFBMjqnZZ2hyBZ7/Pzl8kx7AbLwLABjFiXjtjFvwMbc+17XXnAuFS4cwiQwyPIV3qBdvnjmIj2X2Hb8GvsgC2ex0XD/BVu9Q8UyumMCwSBKqNwneSDKrBNNskqhmcS4NB8wSNpm2ff2mnWl2ERrRqt9VmET5exeCEdiFIsRH3EUYoqouMjRSxI2P7EqNeAf1cKuWyDebN70VJKV6k+CNCHiCaXVyeCdR3UA2btuHyQBV5q0MYWa3y4SxAUHE+IrlC5S5WnTt60rF2SPb4lgJjOPJrpIm4nEPHARzRMATjK/ERxEQ1Q0kiBAQgLMXzUa+p7upYzMqRRdMKeRzuGNgKe68eCmyBpWCGJXsjKqYcVpGtnEPHARzRMATjEkB4CiJ6gYSRAgYwD+/dqilxRxqz3Wgr4Bvr11mNBIfHS/j+k07sdARJknLyDbugYNgjkxiCkpsF4KqaEEYE5g3PrIrUBglagr5nKvzYmikiOs27dR8Wo09AagrcmwfN0lOD3GhJIumcqGEhQh4/Jh8MVvhQhgaKeJ/PPFKZLm+e6wMxsv+lv07HSO6a7Vw3Vbf/bEF2nldaxGnh6BDXCiC8SRaK2K1A30F7Ln9s6Fl6ZvdY02Hmb60fAFmz5zhur9XiMDtWvmJodscGCvVhcRUJDEeLiQbEfAOwlSYw4zVetnmZgUQQyeFfA63XnjadLmyH7243zVWns9ZuPPipXVCfIJV/1Vwu1br15zmu4923H+gr4AX1q3S3riSFg8Xko0IeAdhKsxhuRC8RvzfGnoFY6XGWpp+yFlZrFzSO30cwD3FuJWhaQE+OnE8xDI6Xq7rm9u18hviUE0AitNDCAMR8A5CJw4ZorqRcVguhPVb9mhHsUMjRfyoyco7titn+96DrhVxavffeOkyDPQVPJ9GvARWFwaxsUfYTueQ/URSHCs1jMLF6SH4JT25UISm0U2iTTLXVU/3rNtowNBIUTu6PjBWwsZt+5paiUk4Xhn+egNXiHOCUhdisbfrrlVxrIQP3fy/MckMgn60z4pjOieHnZ+d0RXfeEpcJ+lEBLyDsL+QN+MXS14AAA3dSURBVG7e1VCA1x592vvUCnkQ3Cbj5lates1QGzu3Y986rCzhyNEJLFq3dVqcsprCyfZinYG+AoZ/cwg/enF/g9Dan2PAVcSd5+hVO3OsVK67kbYK543FDnW1uh+CfySE0gHUTiRu3LbPqHp6s7i1Nbh6cdOx3rFSeXpSVBXyscMTs3ssgCv723H46zbt1F6DSebpcNL2vQc9nxIY0K7QdJ6jyfWNw4mSlhWiQiMi4G2OaiJR54AIYwLNvlnohG92j4WBvoJSdP1SO1J0rli9+7IzcM9lZ+D3pQmUfS4YsiczTW9odjilFj8Tl05a7URJywpRoREJobQ5qtGVTs5WLult6liqBUC15Kwsbr2w4gCpjbPbN5UgMXF7pPjCulXKxFG6kbZJm16hmVpqwylZoroRrN0vr4U8NnPzuZbGpHXnKY6Y5CMjcJ+kIR1oLX5yh2zfe7CpY7nFeGf3WA2rDO2ReJaoqQlN1UjRK97sRXGs5PspwRZx+6bhtE0O9BXw+bMK0yEXApDN1I/dnbbIVmQtlNwn6UUE3AdpSQdqMzRS9LXSsdlHZrfPv+dY1j40UsQZtz3jGo8GKiKXz1mVWLYG1UgxrKRXplkIbZxnUjsSHxop4rEdxbpJ0AzqV5HqbJFRxqTjTpomBEdCKD5wm+xJ4h+7X6tes4/MbiGH2uvkFWqx8bLh2Ywfm5i+iboljPKLHZoB0HBcPyGfA2OlSgIvhfunPMVgPu7Mue2pPRgd19svo6JZ15EQDyLgPkjbZI9bv3JWtiFZlekjsy4+u3JJLx5yWZxj98ckvEGojKJXbHhuun1bYNZv2VPnMR8dL2PwkV0AAeXJ8JKz2f11xuttC6KpiOd7LNd4/FipPH0+OvEGJCYtNCIhFB+kbfmzrl/2I3KQR2a3MJJXDN3uj8kNz5Y6VRx55ozGcUd5ikMVb6D++tU6Z2pDIF7krCyY0VQ83m5HYtKCExFwH6Rtssetv3ZSpdc3nN/g4HDDLYzkNeK3r5PfG16pPInbntoz/boVTzy1TwC14Rk/QmzfGA83me8FgMSkBSUi4D5I22RPFP11CyPphDlLVHfcIB7w0fHytJAGeeKxsuRrQlf1BODnxmEv9R/oK7heF7fJWZtCPpfYvzEhXqSgg+ALOxGTE13BgtoiBbWx83yPBa6ukDTFntQ0nQStJZ+zcMGyU1xj9IB+ctJOXmU6QVo7AetWIANonCCtRYo8CIAUdBBCQjd6Hj82AaBxRWSteNfGzkfHy3XpXE2onVS0xc+Uw6Uy7hhYii8tXzBtC8wSYcWH5tT11y2viemTgzOs5vYk5Hxvdo+FfM5KxROeED8yAvdJ2rK2RdHfoZFigxMEcB8t6kbufnDaCv226VZ3srYGqNuxvWpiZolw+TnzcceAvxuMILghI/AQSONCnij6q3OC2JONqpWqfuLHVpZgKVYoOieLVSNiK0Owsupot+78a6+TCmd1ereFPZPMeGxHMbF/E0J7IQLug7RlbYuyvzpBHh0vK28YphOPhXwOGy9Zho2XLvOcfFWFJjZeugwbL1nmq+6km7tEdWyv/CpJ/psQ2gtZyOODdlnIE0Z/TRM92WI2uHoxBh/d5erVri3SAJjlotatIBzoK2DRuq3KmLbz/HXXw9kfm4LBuSf1b0JoL2QE7oN2WcgTRn/9WAHtOpIzu93HC376ZZJUzPT8/V4nk3PP91ipSnompBMRcB+000KeZlGFL/KaCvO2ELotaPG7lN8ktm96/gtPUgu1bnvtuQNo8JdbWcK7702kZq5ESC8SQvFBGLUigxDUSdLq/l6w7BQ8tqOozbGiC7s4F/q4oUsKpUoqZnr+L/56VHks3Xa77dpCxbXHOHJ0osGhE3bSs7S5oYRoEBthwnFbBOL2hY36C67r1+fPKmD73oPK4wY9F7dj1kIAXt9wvu9zWbhuq/a9NwK0p4u9B+2fk2avo5A+dDZCzxE4Ed0P4AIA7zDzR6vb5gDYBGAhgDcArGVm/XBFCEyQFLatKFKr69f2vQeVE3+1xw56Y/HKRRI0tq8rcAxUrqXfaxZ1hZu0pTUWosMkBv4AgM86tq0D8CwzfxjAs9XXQgQEcZKY2gebqS4U1OESNImWV9sEBI7tX37OfO17QeyAUc+VpM0NJUSHp4Az8z8BOOTYfBGAB6v/fxDAQMj9EqoEcZKYfMGbXeQThyPHrW1G8KcLt1WTzmupuuk5twH6lAJhkDY3lBAdQV0oH2DmtwGg+vP9uh2J6BoiGiai4YMHm6u52IkEGc2ZfMGbXeQThyNncPVibUZB3cIdU3Sfr71mqpve4CO7MPjoroYbIYDATxpepM0NJURH5DZCZr6XmfuZub+3t7mq551IkJSwJl/wZh/D40itO9BXwBXLFzSIeBjiZXLNVDc9VSGJqFdipi2tsRAdQW2EvyWiU5j5bSI6BcA7YXaq3WjWEeK3XqHJZGEYE21x1FG8Y2Ap+k+doz23KC2XfmLMUcejpYalAAQX8C0ArgKwofrzydB61Ga0whGiwusLrsvdHfVjuF+B1e2vyyjo51qr2tY5aADz9AH2voIQNSY2wocBfBrAyUT0FoBbURHuzUR0NYD9AC6NspNpJqmWrzAX+ZiKchCBNdnfLRWs7loHubGqbnpWhhqKKYdxI5SFOoIJngLOzJdr3jo35L60JUm2fDlHsrabwo9o+BFCvzczk/1NqvOornWQG6vupqfa1ozYxvXUJqQPWUofMVEv6giLoKLhRwj93sxMtpsUGlZd62Z87Lrsh2GR1Kc2IXlIMquISYvlK6it0I8Q+vUvm2w3eZJRXeske6mT/NQmJAsR8IhJouVLtRglqGj4EUK/NzOT/b0Ed3aPpbzWSbyx2r8XXXaiJNxchGQhIZQWkCTLly5Uku+xMDremO5VJRq1E2yzchasLBlN4vmdODXZXzWxWNuPWy88LXDbunOOIvbtFcuP++YiJBPJRpgQWuU60BUCzucsHJ2Y8sxwpxIaK0M48YQujI2XMTefw8olvdqMhFFQ60KxE1O5FTAO0r6p+yTo05VbgeYwz0VIJ4GzEQrR00rXgS4kcrhUxt2XneF5E9GtRuzp7sLILZ+JxUER9ROO7pydNDPR6LesmyAAIuCJoJWuAzdXjIkQesXK29FB0YoVmGlxKwnJQiYxE0ArXQfNTt55TVq26lyaSYXrFz8iGlRwkzipKiQfEfAE0EpLW7OuGC+hacW5NJsK1y+qc7YyBCtbn1arGcFNoltJSD4yiZkA0lYiy23CtRXnopvwK+RzkcWLo3ahCIIbMomZYOIqlhwUt1h5K87FtGBFmH1oxQpMQfCLCHhCSJJXvFmiPhevCT/JJSJ0ChIDF1KHVxy+2WpDgpAWZAQuREaQMIbJZ7zCNJJLROgURMCFSAgSxvDzGbcwjXiqhU5BQihCJAQJY4QV+hBPtdApyAhciIQgYYywQh9pc/UIQlBEwIVICBLGCDP00U6uHkHQISEUIRKChDEk9CEI/pARuBAJQcIYEvoQBH/IUnpBEISEo1tKLyEUQRCElCICLgiCkFJEwAVBEFKKCLggCEJKEQEXBEFIKS11oRDRQQC/adkBm+NkAL+LuxMRI+fYPnTCeXbyOZ7KzL3OjS0V8DRBRMMq2047IefYPnTCeco5NiIhFEEQhJQiAi4IgpBSRMD13Bt3B1qAnGP70AnnKefoQGLggiAIKUVG4IIgCClFBFwQBCGliIArIKIsEY0Q0dNx9yUqiOgNInqFiHYSUVumiCSiPBE9SkR7iehVIvpPcfcpTIhocfX3Z//7PRFdF3e/woaIrieiPUT0KyJ6mIhOiLtPYUNEX6+e3x4/v0PJB67m6wBeBfC+uDsSMSuZuZ0XRnwXwE+Z+RIi6gbQE3eHwoSZ9wE4A6gMOgAUATwRa6dChogKAL4G4I+ZuUREmwF8AcADsXYsRIjoowD+HMDZAI4B+CkRbWXmf/X6rIzAHRDRPADnA/hB3H0RgkNE7wPwSQD3AQAzH2PmsXh7FSnnAniNmdOy0tkPXQByRNSFyk34QMz9CZuPAHiRmceZeQLALwF8zuSDIuCN3APgGwCm4u5IxDCAZ4hoBxFdE3dnIuCDAA4C+IdqOOwHRDQz7k5FyBcAPBx3J8KGmYsA/hrAfgBvAzjMzM/E26vQ+RWATxLRSUTUA+DPAMw3+aAIeA1EdAGAd5h5R9x9aQErmPlMAOcBuJaIPhl3h0KmC8CZAL7HzH0AjgBYF2+XoqEaHloD4JG4+xI2RDQbwEUAFgGYC2AmEX0p3l6FCzO/CuAuAD8D8FMAuwBMmHxWBLyeFQDWENEbAH4MYBURPRRvl6KBmQ9Uf76DStz07Hh7FDpvAXiLmV+qvn4UFUFvR84D8DIz/zbujkTAnwJ4nZkPMnMZwOMAPh5zn0KHme9j5jOZ+ZMADgHwjH8DIuB1MPPNzDyPmRei8kj6HDO31d0eAIhoJhH9kf1/AJ9B5TGubWDmfwfwJhHZJe3PBfB/Y+xSlFyONgyfVNkPYDkR9RARofJ7fDXmPoUOEb2/+nMBgIth+PsUF0pn8gEAT1S+D+gC8I/M/NN4uxQJ/x3Aj6ohhl8D+GrM/Qmdasz0PwP4i7j7EgXM/BIRPQrgZVTCCiNozyX1jxHRSQDKAK5l5lGTD8lSekEQhJQiIRRBEISUIgIuCIKQUkTABUEQUooIuCAIQkoRARcEQUgpIuCCIAgpRQRcEAQhpfx/uqt5CPq889MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X_rm, y_)\n",
    "plt.scatter(X_rm, [predicate(x, graph_sort) for x in X_rm])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多维向量版本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "\n",
    "class Node:\n",
    "    def __init__(self, inputs=[]):\n",
    "        self.inputs = inputs\n",
    "        self.outputs = []\n",
    "\n",
    "        for n in self.inputs:\n",
    "            n.outputs.append(self)\n",
    "            # set 'self' node as inbound_nodes's outbound_nodes\n",
    "\n",
    "        self.value = None\n",
    "\n",
    "        self.gradients = {}\n",
    "        # keys are the inputs to this node, and their\n",
    "        # values are the partials of this node with \n",
    "        # respect to that input.\n",
    "        # \\partial{node}{input_i}\n",
    "        \n",
    "\n",
    "    def forward(self):\n",
    "        '''\n",
    "        Forward propagation. \n",
    "        Compute the output value vased on 'inbound_nodes' and store the \n",
    "        result in self.value\n",
    "        '''\n",
    "\n",
    "        raise NotImplemented\n",
    "    \n",
    "\n",
    "    def backward(self):\n",
    "\n",
    "        raise NotImplemented\n",
    "        \n",
    "class Placeholder(Node):\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        An Input node has no inbound nodes.\n",
    "        So no need to pass anything to the Node instantiator.\n",
    "        '''\n",
    "        Node.__init__(self)\n",
    "\n",
    "    def forward(self, value=None):\n",
    "        '''\n",
    "        Only input node is the node where the value may be passed\n",
    "        as an argument to forward().\n",
    "        All other node implementations should get the value of the \n",
    "        previous node from self.inbound_nodes\n",
    "        \n",
    "        Example: \n",
    "        val0: self.inbound_nodes[0].value\n",
    "        '''\n",
    "        if value is not None:\n",
    "            self.value = value\n",
    "            ## It's is input node, when need to forward, this node initiate self's value.\n",
    "\n",
    "        # Input subclass just holds a value, such as a data feature or a model parameter(weight/bias)\n",
    "        \n",
    "    def backward(self):\n",
    "        self.gradients = {self:0}\n",
    "        for n in self.outputs:\n",
    "            grad_cost = n.gradients[self]\n",
    "            self.gradients[self] = grad_cost * 1\n",
    "            \n",
    "        \n",
    "        # input N --> N1, N2\n",
    "        # \\partial L / \\partial N \n",
    "        # ==> \\partial L / \\partial N1 * \\ partial N1 / \\partial N\n",
    "\n",
    "\n",
    "class Add(Node):\n",
    "    def __init__(self, *nodes):\n",
    "        Node.__init__(self, nodes)\n",
    "\n",
    "\n",
    "    def forward(self):\n",
    "        self.value = sum(map(lambda n: n.value, self.inputs))\n",
    "        ## when execute forward, this node caculate value as defined.\n",
    "\n",
    "class Linear(Node):\n",
    "    def __init__(self, nodes, weights, bias):\n",
    "        Node.__init__(self, [nodes, weights, bias])\n",
    "\n",
    "    def forward(self):\n",
    "        inputs = self.inputs[0].value\n",
    "        weights = self.inputs[1].value\n",
    "        bias = self.inputs[2].value\n",
    "\n",
    "        self.value = np.dot(inputs, weights) + bias\n",
    "        \n",
    "    def backward(self):\n",
    "\n",
    "# 矩阵运算\n",
    "        # initial a partial for each of the inbound_nodes.\n",
    "        self.gradients = {n: np.zeros_like(n.value) for n in self.inputs}\n",
    "\n",
    "        for n in self.outputs:\n",
    "            # Get the partial of the cost w.r.t this node.\n",
    "            grad_cost = n.gradients[self]\n",
    "\n",
    "            self.gradients[self.inputs[0]] = np.dot(grad_cost, self.inputs[1].value.T)\n",
    "            self.gradients[self.inputs[1]] = np.dot(self.inputs[0].value.T, grad_cost)\n",
    "            self.gradients[self.inputs[2]] = np.sum(grad_cost, axis=0, keepdims=False)\n",
    "\n",
    "        # WX + B / W ==> X\n",
    "        # WX + B / X ==> W\n",
    "\n",
    "class Sigmoid(Node):\n",
    "    def __init__(self, node):\n",
    "        Node.__init__(self, [node])\n",
    "\n",
    "\n",
    "    def _sigmoid(self, x):\n",
    "        return 1./(1 + np.exp(-1 * x))\n",
    "\n",
    "    def forward(self):\n",
    "        self.x = self.inputs[0].value\n",
    "        self.value = self._sigmoid(self.x)\n",
    "\n",
    "    def backward(self):\n",
    "        self.partial = self._sigmoid(self.x) * (1 - self._sigmoid(self.x))\n",
    "        \n",
    "        # y = 1 / (1 + e^-x)\n",
    "        # y' = 1 / (1 + e^-x) (1 - 1 / (1 + e^-x))\n",
    "        \n",
    "# 矩阵运算        \n",
    "        self.gradients = {n: np.zeros_like(n.value) for n in self.inputs}\n",
    "\n",
    "        for n in self.outputs:\n",
    "            grad_cost = n.gradients[self]  # Get the partial of the cost with respect to this node.\n",
    "\n",
    "            self.gradients[self.inputs[0]] = grad_cost * self.partial\n",
    "            # use * to keep all the dimension same!.\n",
    "\n",
    "\n",
    "\n",
    "class MSE(Node):\n",
    "    def __init__(self, y, a):\n",
    "        Node.__init__(self, [y, a])\n",
    "\n",
    "\n",
    "    def forward(self):\n",
    "        y = self.inputs[0].value.reshape(-1, 1)\n",
    "        a = self.inputs[1].value.reshape(-1, 1)\n",
    "        assert(y.shape == a.shape)\n",
    "\n",
    "        self.m = self.inputs[0].value.shape[0]\n",
    "        self.diff = y - a\n",
    "\n",
    "        self.value = np.mean(self.diff**2)\n",
    "\n",
    "\n",
    "    def backward(self):\n",
    "        self.gradients[self.inputs[0]] = (2 / self.m) * self.diff\n",
    "        self.gradients[self.inputs[1]] = (-2 / self.m) * self.diff\n",
    "\n",
    "\n",
    "def forward_and_backward(graph):\n",
    "    # execute all the forward method of sorted_nodes.\n",
    "\n",
    "    ## In practice, it's common to feed in mutiple data example in each forward pass rather than just 1. Because the examples can be processed in parallel. The number of examples is called batch size.\n",
    "    for n in graph:\n",
    "        n.forward()\n",
    "        ## each node execute forward, get self.value based on the topological sort result.\n",
    "\n",
    "    for n in  graph[::-1]:\n",
    "        n.backward()\n",
    "\n",
    "###   v -->  a -->  C\n",
    "##    b --> C\n",
    "##    b --> v -- a --> C\n",
    "##    v --> v ---> a -- > C\n",
    "\n",
    "def toplogic(graph):\n",
    "    sorted_node = []\n",
    "    \n",
    "    while len(graph) > 0: \n",
    "\n",
    "        all_inputs = []\n",
    "        all_outputs = []\n",
    "        \n",
    "        for n in graph:\n",
    "            all_inputs += graph[n]\n",
    "            all_outputs.append(n)\n",
    "        \n",
    "        all_inputs = set(all_inputs)\n",
    "        all_outputs = set(all_outputs)\n",
    "    \n",
    "        need_remove = all_outputs - all_inputs  # which in all_inputs but not in all_outputs\n",
    "    \n",
    "        if len(need_remove) > 0: \n",
    "            node = random.choice(list(need_remove))\n",
    "\n",
    "            need_to_visited = [node]\n",
    "\n",
    "            if len(graph) == 1: need_to_visited += graph[node]\n",
    "                \n",
    "            graph.pop(node)\n",
    "            sorted_node += need_to_visited\n",
    "        \n",
    "            for _, links in graph.items():\n",
    "                if node in links: links.remove(node)\n",
    "        else: # have cycle\n",
    "            break\n",
    "        \n",
    "    return sorted_node\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "def convert_feed_dict_to_graph(feed_dict):\n",
    "    computing_graph = defaultdict(list)\n",
    "    \n",
    "    nodes = [n for n in feed_dict]\n",
    "    \n",
    "    while nodes:\n",
    "        n = nodes.pop(0) \n",
    "        \n",
    "        if isinstance(n, Placeholder):\n",
    "            n.value = feed_dict[n]\n",
    "        \n",
    "        if n in computing_graph: continue\n",
    "\n",
    "        for m in n.outputs:\n",
    "            computing_graph[n].append(m)\n",
    "            nodes.append(m)\n",
    "    \n",
    "    return computing_graph\n",
    "\n",
    "def topological_sort_feed_dict(feed_dict):\n",
    "    graph = convert_feed_dict_to_graph(feed_dict)\n",
    "    \n",
    "    return toplogic(graph)\n",
    "\n",
    "\n",
    "def optimize(trainables, learning_rate=1e-2):\n",
    "    # there are so many other update / optimization methods\n",
    "    # such as Adam, Mom, \n",
    "    for t in trainables:\n",
    "        t.value += -1 * learning_rate * t.gradients[t]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> 十几维:拟合效果很好(loss=2左右),但是无法确定具体的函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of examples = 506\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.utils import shuffle, resample\n",
    "#from miniflow import *\n",
    "\n",
    "# Load data\n",
    "data = load_boston()\n",
    "X_ = data['data']\n",
    "y_ = data['target']\n",
    "\n",
    "# Normalize data\n",
    "X_ = (X_ - np.mean(X_, axis=0)) / np.std(X_, axis=0)\n",
    "\n",
    "n_features = X_.shape[1]\n",
    "n_hidden = 10\n",
    "W1_ = np.random.randn(n_features, n_hidden)\n",
    "b1_ = np.zeros(n_hidden)\n",
    "W2_ = np.random.randn(n_hidden, 1)\n",
    "b2_ = np.zeros(1)\n",
    "\n",
    "# Neural network\n",
    "X, y = Placeholder(), Placeholder()\n",
    "W1, b1 = Placeholder(), Placeholder()\n",
    "W2, b2 = Placeholder(), Placeholder()\n",
    "\n",
    "l1 = Linear(X, W1, b1)\n",
    "s1 = Sigmoid(l1)\n",
    "l2 = Linear(s1, W2, b2)\n",
    "cost = MSE(y, l2)\n",
    "\n",
    "feed_dict = {\n",
    "    X: X_,\n",
    "    y: y_,\n",
    "    W1: W1_,\n",
    "    b1: b1_,\n",
    "    W2: W2_,\n",
    "    b2: b2_\n",
    "}\n",
    "\n",
    "epochs = 5000\n",
    "# Total number of examples\n",
    "m = X_.shape[0]\n",
    "batch_size = 16\n",
    "steps_per_epoch = m // batch_size\n",
    "\n",
    "graph = topological_sort_feed_dict(feed_dict)\n",
    "trainables = [W1, b1, W2, b2]\n",
    "\n",
    "print(\"Total number of examples = {}\".format(m))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, Loss: 156.649\n",
      "Epoch: 101, Loss: 6.483\n",
      "Epoch: 201, Loss: 5.479\n",
      "Epoch: 301, Loss: 5.114\n",
      "Epoch: 401, Loss: 4.100\n",
      "Epoch: 501, Loss: 3.968\n",
      "Epoch: 601, Loss: 3.872\n",
      "Epoch: 701, Loss: 3.966\n",
      "Epoch: 801, Loss: 4.467\n",
      "Epoch: 901, Loss: 4.511\n",
      "Epoch: 1001, Loss: 3.843\n",
      "Epoch: 1101, Loss: 3.958\n",
      "Epoch: 1201, Loss: 3.830\n",
      "Epoch: 1301, Loss: 3.525\n",
      "Epoch: 1401, Loss: 3.508\n",
      "Epoch: 1501, Loss: 3.301\n",
      "Epoch: 1601, Loss: 3.796\n",
      "Epoch: 1701, Loss: 3.843\n",
      "Epoch: 1801, Loss: 3.532\n",
      "Epoch: 1901, Loss: 4.315\n",
      "Epoch: 2001, Loss: 3.670\n",
      "Epoch: 2101, Loss: 3.705\n",
      "Epoch: 2201, Loss: 3.241\n",
      "Epoch: 2301, Loss: 3.281\n",
      "Epoch: 2401, Loss: 3.861\n",
      "Epoch: 2501, Loss: 3.112\n",
      "Epoch: 2601, Loss: 3.067\n",
      "Epoch: 2701, Loss: 3.402\n",
      "Epoch: 2801, Loss: 3.601\n",
      "Epoch: 2901, Loss: 3.645\n",
      "Epoch: 3001, Loss: 3.038\n",
      "Epoch: 3101, Loss: 3.726\n",
      "Epoch: 3201, Loss: 3.638\n",
      "Epoch: 3301, Loss: 3.752\n",
      "Epoch: 3401, Loss: 3.383\n",
      "Epoch: 3501, Loss: 3.524\n",
      "Epoch: 3601, Loss: 3.942\n",
      "Epoch: 3701, Loss: 3.395\n",
      "Epoch: 3801, Loss: 3.622\n",
      "Epoch: 3901, Loss: 3.489\n",
      "Epoch: 4001, Loss: 3.472\n",
      "Epoch: 4101, Loss: 3.466\n",
      "Epoch: 4201, Loss: 3.610\n",
      "Epoch: 4301, Loss: 3.512\n",
      "Epoch: 4401, Loss: 3.604\n",
      "Epoch: 4501, Loss: 3.373\n",
      "Epoch: 4601, Loss: 3.423\n",
      "Epoch: 4701, Loss: 3.900\n",
      "Epoch: 4801, Loss: 3.600\n",
      "Epoch: 4901, Loss: 3.529\n"
     ]
    }
   ],
   "source": [
    "losses = []\n",
    "\n",
    "for i in range(epochs):\n",
    "    loss = 0\n",
    "    for j in range(steps_per_epoch):\n",
    "        # Step 1\n",
    "        # Randomly sample a batch of examples\n",
    "        X_batch, y_batch = resample(X_, y_, n_samples=batch_size)\n",
    "\n",
    "        # Reset value of X and y Inputs\n",
    "        X.value = X_batch\n",
    "        y.value = y_batch\n",
    "\n",
    "        # Step 2\n",
    "        _ = None\n",
    "        forward_and_backward(graph) # set output node not important.\n",
    "\n",
    "        # Step 3\n",
    "        rate = 1e-2\n",
    "    \n",
    "        optimize(trainables, rate)\n",
    "\n",
    "        loss += graph[-1].value\n",
    "        \n",
    "    \n",
    "    if i % 100 == 0: \n",
    "        print(\"Epoch: {}, Loss: {:.3f}\".format(i+1, loss/steps_per_epoch))\n",
    "        losses.append(loss/steps_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fef1702db10>]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAbrUlEQVR4nO3de2xk533e8e8zc4Yz5N64F+5aWq69sry+KIFsC6yg1E3hSGkjO4alP2xARtIsUgGLpmriNEltOQFqtIAAuxc7SS8GtrZqGXBlC44dCYGbWpHtugViuZRt3SwrWkuyltrVLvfCXS2vc/n1j3NmOOSSu1wOKWrOPB+AmJkzZzjvIc8885v3PWdeRQRmZpYvhY1ugJmZrT2Hu5lZDjnczcxyyOFuZpZDDnczsxxKNroBALt27Yr9+/dvdDPMzLrKY489dioihpa673UR7vv372d0dHSjm2Fm1lUk/Xy5+9wtY2aWQ5cNd0n3Sjop6alFy39X0rOSnpb079qWf0LSkey+X1uPRpuZ2aWtpFvmi8B/Br7UXCDpV4DbgOsjYlbS7mz5dcAdwC8AVwN/I+mtEVFf64abmdnyLlu5R8T3gDOLFv8O8KmImM3WOZktvw34SkTMRsQLwBHgxjVsr5mZrcBq+9zfCvyypEcl/W9Jfy9bvhc42rbeWLbsIpIOSRqVNDo+Pr7KZpiZ2VJWG+4JsB24CfhXwAOSBGiJdZf8ZrKIOBwRIxExMjS05JE8Zma2SqsN9zHg65H6AdAAdmXL97WtNwwc66yJZmZ2pVYb7n8J3Awg6a1AH3AKeAi4Q1JZ0jXAAeAHa9HQpTz7yqv8x289y+kLs+v1FGZmXWklh0LeD/wt8DZJY5LuBO4F3pwdHvkV4GBWxT8NPAD8BPhr4K71PFLmZ+MX+E/fPsK4w93MbIHLHgoZER9Z5q7fXGb9e4B7OmnUSlVK6XvTbLXxWjydmVnX6OozVMtJEYCZqg+jNzNr1+XhnlXuNVfuZmbtujzc08rd4W5mtlBXh3urz73mbhkzs3ZdHe6tyt0DqmZmC3R3uGeV+4wrdzOzBbo73BMfCmlmtpQuD3cPqJqZLaXLw90DqmZmS+nqcC8URF+xwIy7ZczMFujqcIe0enflbma2UPeHe6ngPnczs0W6P9yToo+WMTNbpPvDveRuGTOzxbo/3JOiB1TNzBbJQbi7cjczW6zrw73iAVUzs4usZJq9eyWdzKbUW3zfH0kKSbuy25L055KOSHpC0g3r0eh25aTocDczW2QllfsXgVsXL5S0D/hHwEtti99HOin2AeAQ8LnOm3hp5aTArGdiMjNb4LLhHhHfA84scddngY8B0bbsNuBL2WTZ3wcGJV21Ji1dRrnkyt3MbLFV9blL+iDwckQ8vuiuvcDRtttj2bKlfschSaOSRsfHx1fTDMCVu5nZUq443CUNAH8C/Oul7l5iWSyxjIg4HBEjETEyNDR0pc1o8YCqmdnFklU85lrgGuBxSQDDwA8l3Uhaqe9rW3cYONZpIy8lPc7dlbuZWbsrrtwj4smI2B0R+yNiP2mg3xARrwAPAb+VHTVzE3AuIo6vbZMXSo9zd+VuZtZuJYdC3g/8LfA2SWOS7rzE6t8EngeOAP8N+Odr0spLKCdFao2gVnfAm5k1XbZbJiI+cpn797ddD+Cuzpu1cpVsHtW5eoOk2PXnZJmZrYmuT0PPo2pmdrHuD/dSOo/qjL9fxsyspfvD3ZW7mdlFchDuaeXuI2bMzOZ1fbg3B1T9tb9mZvO6Ptyblbsn7DAzm9f94e7K3czsIt0f7h5QNTO7SNeHe6XkAVUzs8W6Ptxblbu7ZczMWnIQ7h5QNTNbLAfh7srdzGyxrg9397mbmV2s68O9z0fLmJldpOvDvVgQpaL8xWFmZm26PtwhHVR15W5mNm8lMzHdK+mkpKfalv17ST+V9ISkb0gabLvvE5KOSHpW0q+tV8PbpVPtuXI3M2taSeX+ReDWRcseBn4xIq4H/g74BICk64A7gF/IHvNfJRXXrLXLqJSKHlA1M2tz2XCPiO8BZxYt+1ZE1LKb3weGs+u3AV+JiNmIeIF0LtUb17C9SyonBWaqrtzNzJrWos/9nwL/M7u+Fzjadt9Ytuwikg5JGpU0Oj4+3lED+pKCK3czszYdhbukPwFqwJebi5ZYLZZ6bEQcjoiRiBgZGhrqpBmU3S1jZrZAstoHSjoIfAC4JSKaAT4G7GtbbRg4tvrmrUwlKTDrbhkzs5ZVVe6SbgU+DnwwIqba7noIuENSWdI1wAHgB50389JcuZuZLXTZyl3S/cB7gV2SxoBPkh4dUwYelgTw/Yj4ZxHxtKQHgJ+QdtfcFRHrXlJ7QNXMbKHLhntEfGSJxV+4xPr3APd00qgrVU4KzLlyNzNryc8Zqg53M7OWXIR7peQzVM3M2uUi3MtJ0ZN1mJm1yUe4u3I3M1sgH+GeFKjWg3pjyfOlzMx6Ti7CvTkbk4+YMTNL5SLcPY+qmdlCOQn3tHL3oKqZWSon4e7K3cysXS7Cvdnn7hOZzMxSuQj3VuXubhkzMyAv4V5KN2PG3TJmZkBewj0bUHXlbmaWykm4e0DVzKxdLsLdA6pmZgvlItyblbsn7DAzS+Uj3EvNbhlX7mZmsIJwl3SvpJOSnmpbtkPSw5Keyy63Z8sl6c8lHZH0hKQb1rPxTfMDqq7czcxgZZX7F4FbFy27G3gkIg4Aj2S3Ad5HOin2AeAQ8Lm1aealVVy5m5ktcNlwj4jvAWcWLb4NuC+7fh9we9vyL0Xq+8CgpKvWqrHL6Ss63M3M2q22z31PRBwHyC53Z8v3Akfb1hvLll1E0iFJo5JGx8fHV9mMVFIskBTkAVUzs8xaD6hqiWVLzqAREYcjYiQiRoaGhjp+4nJScOVuZpZZbbifaHa3ZJcns+VjwL629YaBY6tv3spVSkWfxGRmllltuD8EHMyuHwQebFv+W9lRMzcB55rdN+utnBT89QNmZpnkcitIuh94L7BL0hjwSeBTwAOS7gReAj6crf5N4P3AEWAK+O11aPOSyqUiM+6WMTMDVhDuEfGRZe66ZYl1A7ir00atRlq5u1vGzAxycoYqeEDVzKxdfsLdA6pmZi35CXdX7mZmLTkK9yIzPlrGzAzIU7iXCu6WMTPL5CbcK0nRx7mbmWVyE+5p5e5wNzODPIW7j3M3M2vJUbgXXbmbmWVyFO4F5uoNGo0lv4TSzKyn5CbcK6V0qr25uqt3M7PchHs5yWZj8hEzZmY5CvdsHtUZH+tuZpajcE/SbhlX7mZmOQr3Sqk5SbYrdzOz3IR7q3L34ZBmZnkK96zP3ScymZl1Fu6S/qWkpyU9Jel+SRVJ10h6VNJzkr4qqW+tGnspraNlXLmbma0+3CXtBX4PGImIXwSKwB3Ap4HPRsQB4Cxw51o09HKax7m7z93MrPNumQTol5QAA8Bx4Gbga9n99wG3d/gcK9I8FNJHy5iZdRDuEfEy8B+Al0hD/RzwGDAREbVstTFg71KPl3RI0qik0fHx8dU2o6U5oOrj3M3MOuuW2Q7cBlwDXA1sAt63xKpLftlLRByOiJGIGBkaGlptM1p8hqqZ2bxOumV+FXghIsYjogp8Hfj7wGDWTQMwDBzrsI0r4gFVM7N5nYT7S8BNkgYkCbgF+AnwHeBD2ToHgQc7a+LKeEDVzGxeJ33uj5IOnP4QeDL7XYeBjwN/IOkIsBP4whq087LcLWNmNi+5/CrLi4hPAp9ctPh54MZOfu9qJMUCxYI8oGpmRo7OUIXmVHuu3M3MchXulZKn2jMzg5yFezkpeEDVzIwchvuMu2XMzPIW7kVX7mZm5C3cSwX3uZuZkbNwryRFHy1jZkbOwj2t3N0tY2aWr3D3gKqZGZC7cPeAqpkZ5C3cPaBqZgbkLdwTn6FqZga5C/cCM1V3y5iZ5Svc3S1jZgbkLNwrSZG5WoOIJWf2MzPrGbkK93LJU+2ZmUGH4S5pUNLXJP1U0jOSfknSDkkPS3ouu9y+Vo29nHKSTbXnY93NrMd1Wrn/GfDXEfF24J3AM8DdwCMRcQB4JLv9mpifJNuDqmbW21Yd7pK2Av+QbI7UiJiLiAngNuC+bLX7gNs7beRKzYe7K3cz622dVO5vBsaB/y7pR5I+L2kTsCcijgNkl7vXoJ0rUill3TKu3M2sx3US7glwA/C5iHg3MMkVdMFIOiRpVNLo+Ph4B82Y16zc/f0yZtbrOgn3MWAsIh7Nbn+NNOxPSLoKILs8udSDI+JwRIxExMjQ0FAHzZhXduVuZgZ0EO4R8QpwVNLbskW3AD8BHgIOZssOAg921MIr0Opzd+VuZj0u6fDxvwt8WVIf8Dzw26RvGA9IuhN4Cfhwh8+xYvN97g53M+ttHYV7RPwYGFnirls6+b2r5UMhzcxS+TpD1QOqZmZA3sLdA6pmZkDewt0nMZmZATkL99aAqrtlzKzH5SrcPaBqZpbKVbgnBVGQB1TNzHIV7pKyeVRduZtZb8tVuANUPNWemVn+wr2cFD2gamY9L3/hXiow424ZM+tx+Qv3pODK3cx6Xu7CvVLygKqZWe7CvZx4QNXMLIfhXnS4m1nPy2G4F5ipulvGzHpb/sLdx7mbmeUv3Cs+Q9XMrPNwl1SU9CNJf5XdvkbSo5Kek/TVbAq+10y55EMhzczWonL/KPBM2+1PA5+NiAPAWeDONXiOFSsnRfe5m1nP6yjcJQ0Dvw58Prst4Gbga9kq9wG3d/IcV8qHQpqZdV65/ynwMaCZpjuBiYioZbfHgL1LPVDSIUmjkkbHx8c7bMa8cik9FDIi1ux3mpl1m1WHu6QPACcj4rH2xUusumTKRsThiBiJiJGhoaHVNuMizQk75uqu3s2sdyUdPPY9wAclvR+oAFtJK/lBSUlWvQ8Dxzpv5so1w32m2qCcFF/LpzYze91YdeUeEZ+IiOGI2A/cAXw7In4D+A7woWy1g8CDHbfyCpSb86j6cEgz62HrcZz7x4E/kHSEtA/+C+vwHMtqzaPqwyHNrId10i3TEhHfBb6bXX8euHEtfu9qVFqVu8PdzHpX7s5QbVXu7pYxsx6W23CfcbeMmfWwHIa7B1TNzHIX7pVSs1vGlbuZ9a7chXurcne3jJn1sPyFe8kDqmZm+Qt3H+duZpa/cK/4DFUzs/yF+/xx7q7czax35TDcfYaqmVnuwr1UFBKejcnMelruwl2SZ2Mys56Xu3CHdFB11pW7mfWwXIa7K3cz63U5Dfei+9zNrKflNNxduZtZb8tluFdKRYe7mfW0VYe7pH2SviPpGUlPS/potnyHpIclPZddbl+75q5MWrm7W8bMelcnlXsN+MOIeAdwE3CXpOuAu4FHIuIA8Eh2+zVVLhU8WYeZ9bRVh3tEHI+IH2bXXwWeAfYCtwH3ZavdB9zeaSOvVDkpunI3s562Jn3ukvYD7wYeBfZExHFI3wCA3cs85pCkUUmj4+Pja9GMlnJS8LdCmllP6zjcJW0G/gL4/Yg4v9LHRcThiBiJiJGhoaFOm7GAB1TNrNd1FO6SSqTB/uWI+Hq2+ISkq7L7rwJOdtbEK+cBVTPrdZ0cLSPgC8AzEfGZtrseAg5m1w8CD66+eatTTjygama9Lengse8B/gnwpKQfZ8v+GPgU8ICkO4GXgA931sQrVy55QNXMetuqwz0i/i+gZe6+ZbW/dy1UsjNUI4L0A4aZWW/J5Rmq5VKRCKjWY6ObYma2IfIZ7tlUezPumjGzHpXrcPex7mbWq/IZ7qXmPKqu3M2sN+Uz3JuVu09kMrMeldNwzyp3d8uYWY/KZ7iXPKBqZr0tn+HuAVUz63G5DPeKB1TNrMflMtw9oGpmvS6n4Z5W7mcn5za4JWZmG6OTLw573RocKAFw99ef5DMP/x3XDw/yrn3buH54kOuHtzE40LfBLTQzW1+5DPddm8t88/d+mUdfOM0TY+d4/OgEf/PMidb92/pLbB8oMTjQx45NfQwOlNg+0Mfw9n7euW+Q667a2uq3NzPrRrkMd4Drrt7KdVdvbd0+N13lqZfP8cTYOY6fm+bsVJWJqTlOnJ/h2Vde5czkHNPVdAA2KYi3X7WFdw4P8s59g7x1zxb6S0X6kkL6U0wvy0mBpCCKBV322ycjgmo9mJia45XzM5w4P8sr52c4eX6GV87NcH6mSiPS9eqNoBHQiKBULDC8vZ837dzEm3YMsH/XAMPbB5Z882k+diXteT2ardV54dQkz524wCvnZtizrcIbdwzwph0DDA6UunKbzDaKIjb+mxNHRkZidHR0o5vBK+dmeHxsgsePTvD42ARPHD3Hq7O1FT22r1ggKYqkIErFAvUIavWgWm9Qa6Shu5SCYGhLmW39JYqFAgVBQaJQEAWlh3MePTO1oB0SDG0uI6XffDlXazBXb1CtN2j+O0tFtd6Emj+b+hK2Vkps7W9elthaSejvS5DS729OL4WydvT3FRnoK9JfKjLQl9DfV6ScFDg7NceJ87OcOD/T9jPLTLVOI3uTiYB6ROtNamulxLb+9Gdrf8K2/hL1Bvxs/AJHTl7g56cnWebPxJZywr4dAwxv76cvKRDZm1/zshGwpZKwe0uZobaf3VvKbC6XCLI3zMb8Y6bm6hybmObliWmOTUwzNjHNy2enOT05y97Bft48tJlrhzZz7dAmrh3azNWD/RQLy7/BRAQXZmtMTFU5N53+vDpTZWquznS1zvRcvXV9plpP/z6NaLW/+VrcvbXC8PZ+hrf3s2/7AFdtq5AUC0QEpy7M8eLpSV48NZlenp5icrZGUhBJYX4fTIoFBFTrjXQfyfaPar1Bo5F2XaafXNNPrdsH0k+wjUjfZOdqDWZrjeyyzsRUldMX5jg9Ocup7PL0hTkEvGFbhasH+7kqu7x6Wz/bBkpMz9WZnKsxNZtdzqV/g6QoykmxVSA199NCtgMu3g+X+4vXI1rtnK02mKvXma2mr7fm36CvmF42/y61RlCvB9VGUMtem7V6UC4V2NSX7uObykU2lRM29SUUC6IekRVOZPt1MDlX5+SrM5w8P9t2Ocv5mWr6OsteW+lliS2VJNtGtQrCpJC+zm9443Z+6dqdl4qXZUl6LCJGlrzP4b68RiN4/tQkL5yabO3wc7X0BTKbBWqtnu4kc/X5naVab2T/vAKlorIdK72+rb/Enq0V9myt8IZtFXZu6iMpXnpcOyI4O1XlxdOTvHR6ihdPT3JsYpqC0jeSUrFAKUnDPCkUWjt9s63N8J+crXF+psr56eZllVdna3S6C6TbVGb3lgoDfUWKBS14gypKzNYbnJ9On/PcdJXzMzXOTVcpCPbv3MSBPZt5y9Bmrt29mQO7t7B3sJ9Xzs/w0pkpXjozxdEzU/z89CRjZ6epN6L15lNQFgASr85UOfnqLHOrOEqqLymwd7CfvYP97NjUx8sT0xw5eYFz09UF6wz0FSkq/WRUEK1tnanWmZiuLvsm3q5UFJWkOP/3Kcz/vkbAqQuzC/4nxYLYs6XM+ZkaF9re5IsFsW97P9v6S1TrQa0xH1a1eoMASllwlrKgK2X72rnpKmen5jg7tbI2JwWxc3MfOzeV2bm5j12by+zc1Ec9guMTMxw/N82xczMXtb1df6lIpVSgVg9ms/1yLRULopwU0kDO/g5z9eWfI30DSF+bzdfIldpSThjaWmbPlgq7t6ZF2uRsvfX6Oj9TS19nM9X0f5MVeu1/899577V8/Na3r2qbLxXuue2WWQuFgnjL7s28ZffmDW2HJHZsSscHbnjj9jX93Y1GpBObkFa0QfpmEtl909V6q+KamqszNVdjptpgx6Y+9mwts2drZdXjE5FVrMtVw9sGSrztDVuu+Heen64xfmG+mpqcq1FseyMoKK2c+pICV2eBvnNTH4VF7YgIzkzO8bPxSZ4fv8ALpyaZzj6ZND8FNLKKrlIqpNVwfx/bBkoMZp9QtlRKDGSffirZJ6DSZd7M52oNjp+bZuzsNGNnpxg7m36i2NpfYv/OAd60axPX7NzE3u39l/1dl9NoBK/O1jg7OcfEdJUk+7uUk2bXYzH71FdcUbfYXK3BifMznJuuMtCXVcDlhP5S8aL/c0Qavs1CpBEQpDthuh9mt1vrL3yugtIwL5cK2Sfni/8Wza7KZrAmBS3blTpXazA1V2Nyrs7UbPpG2sgm/GnuP4VC9om2VGT31jIDfauL0PZ2Fdapu3HdKndJtwJ/BhSBz0fEp5Zb9/VauZuZvZ5dqnJfl+PcJRWB/wK8D7gO+Iik69bjuczM7GLrdRLTjcCRiHg+IuaArwC3rdNzmZnZIusV7nuBo223x7JlLZIOSRqVNDo+Pr5OzTAz603rFe5LjRAs6NyPiMMRMRIRI0NDQ+vUDDOz3rRe4T4G7Gu7PQwcW6fnMjOzRdYr3P8fcEDSNZL6gDuAh9bpuczMbJF1Oc49ImqS/gXwv0gPhbw3Ip5ej+cyM7OLrdtJTBHxTeCb6/X7zcxsea+Lrx+QNA78fJUP3wWcWsPmdJNe3XZvd2/xdi/vTRGx5BEpr4tw74Sk0eXO0Mq7Xt12b3dv8XavTi5nYjIz63UOdzOzHMpDuB/e6AZsoF7ddm93b/F2r0LX97mbmdnF8lC5m5nZIg53M7Mc6upwl3SrpGclHZF090a3Z71IulfSSUlPtS3bIelhSc9ll2s7RdPrgKR9kr4j6RlJT0v6aLY819suqSLpB5Iez7b732TLr5H0aLbdX82+2iN3JBUl/UjSX2W3c7/dkl6U9KSkH0sazZZ1tJ93bbj32IQgXwRuXbTsbuCRiDgAPJLdzpsa8IcR8Q7gJuCu7H+c922fBW6OiHcC7wJulXQT8Gngs9l2nwXu3MA2rqePAs+03e6V7f6ViHhX27HtHe3nXRvu9NCEIBHxPeDMosW3Afdl1+8Dbn9NG/UaiIjjEfHD7PqrpC/4veR82yN1IbtZyn4CuBn4WrY8d9sNIGkY+HXg89lt0QPbvYyO9vNuDvfLTgiSc3si4jikIQjs3uD2rCtJ+4F3A4/SA9uedU38GDgJPAz8DJiIiFq2Sl739z8FPgY0sts76Y3tDuBbkh6TdChb1tF+vm5fHPYauOyEIJYPkjYDfwH8fkScXzxrfR5FRB14l6RB4BvAO5Za7bVt1fqS9AHgZEQ8Jum9zcVLrJqr7c68JyKOSdoNPCzpp53+wm6u3Ht9QpATkq4CyC5PbnB71oWkEmmwfzkivp4t7oltB4iICeC7pGMOg5KaBVke9/f3AB+U9CJpN+vNpJV83rebiDiWXZ4kfTO/kQ73824O916fEOQh4GB2/SDw4Aa2ZV1k/a1fAJ6JiM+03ZXrbZc0lFXsSOoHfpV0vOE7wIey1XK33RHxiYgYjoj9pK/nb0fEb5Dz7Za0SdKW5nXgHwNP0eF+3tVnqEp6P+k7e3NCkHs2uEnrQtL9wHtJvwL0BPBJ4C+BB4A3Ai8BH46IxYOuXU3SPwD+D/Ak832wf0za757bbZd0PekAWpG0AHsgIv6tpDeTVrQ7gB8BvxkRsxvX0vWTdcv8UUR8IO/bnW3fN7KbCfA/IuIeSTvpYD/v6nA3M7OldXO3jJmZLcPhbmaWQw53M7MccribmeWQw93MLIcc7mZmOeRwNzPLof8Plsu2Tc07UI0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2-Dimensions Version\n",
    "> 便于观察大致函数形态"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe = pd.DataFrame(data['data'])\n",
    "dataframe.columns = data['feature_names']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>501</th>\n",
       "      <td>0.06263</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573</td>\n",
       "      <td>6.593</td>\n",
       "      <td>69.1</td>\n",
       "      <td>2.4786</td>\n",
       "      <td>1.0</td>\n",
       "      <td>273.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>391.99</td>\n",
       "      <td>9.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>502</th>\n",
       "      <td>0.04527</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573</td>\n",
       "      <td>6.120</td>\n",
       "      <td>76.7</td>\n",
       "      <td>2.2875</td>\n",
       "      <td>1.0</td>\n",
       "      <td>273.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503</th>\n",
       "      <td>0.06076</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573</td>\n",
       "      <td>6.976</td>\n",
       "      <td>91.0</td>\n",
       "      <td>2.1675</td>\n",
       "      <td>1.0</td>\n",
       "      <td>273.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>504</th>\n",
       "      <td>0.10959</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573</td>\n",
       "      <td>6.794</td>\n",
       "      <td>89.3</td>\n",
       "      <td>2.3889</td>\n",
       "      <td>1.0</td>\n",
       "      <td>273.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>393.45</td>\n",
       "      <td>6.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>505</th>\n",
       "      <td>0.04741</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573</td>\n",
       "      <td>6.030</td>\n",
       "      <td>80.8</td>\n",
       "      <td>2.5050</td>\n",
       "      <td>1.0</td>\n",
       "      <td>273.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>396.90</td>\n",
       "      <td>7.88</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>506 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0    0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1    0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2    0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3    0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4    0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "..       ...   ...    ...   ...    ...    ...   ...     ...  ...    ...   \n",
       "501  0.06263   0.0  11.93   0.0  0.573  6.593  69.1  2.4786  1.0  273.0   \n",
       "502  0.04527   0.0  11.93   0.0  0.573  6.120  76.7  2.2875  1.0  273.0   \n",
       "503  0.06076   0.0  11.93   0.0  0.573  6.976  91.0  2.1675  1.0  273.0   \n",
       "504  0.10959   0.0  11.93   0.0  0.573  6.794  89.3  2.3889  1.0  273.0   \n",
       "505  0.04741   0.0  11.93   0.0  0.573  6.030  80.8  2.5050  1.0  273.0   \n",
       "\n",
       "     PTRATIO       B  LSTAT  \n",
       "0       15.3  396.90   4.98  \n",
       "1       17.8  396.90   9.14  \n",
       "2       17.8  392.83   4.03  \n",
       "3       18.7  394.63   2.94  \n",
       "4       18.7  396.90   5.33  \n",
       "..       ...     ...    ...  \n",
       "501     21.0  391.99   9.67  \n",
       "502     21.0  396.90   9.08  \n",
       "503     21.0  396.90   5.64  \n",
       "504     21.0  393.45   6.48  \n",
       "505     21.0  396.90   7.88  \n",
       "\n",
       "[506 rows x 13 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data = dataframe[['RM', 'LSTAT']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of examples = 506\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.utils import shuffle, resample\n",
    "#from miniflow import *\n",
    "\n",
    "# Load data\n",
    "data = load_boston()\n",
    "X_ = training_data\n",
    "y_ = data['target']\n",
    "\n",
    "# Normalize data\n",
    "X_ = (X_ - np.mean(X_, axis=0)) / np.std(X_, axis=0)\n",
    "\n",
    "n_features = X_.shape[1]\n",
    "n_hidden = 10\n",
    "W1_ = np.random.randn(n_features, n_hidden)\n",
    "b1_ = np.zeros(n_hidden)\n",
    "W2_ = np.random.randn(n_hidden, 1)\n",
    "b2_ = np.zeros(1)\n",
    "\n",
    "# Neural network\n",
    "X, y = Placeholder(), Placeholder()\n",
    "W1, b1 = Placeholder(), Placeholder()\n",
    "W2, b2 = Placeholder(), Placeholder()\n",
    "\n",
    "l1 = Linear(X, W1, b1)\n",
    "s1 = Sigmoid(l1)\n",
    "l2 = Linear(s1, W2, b2)\n",
    "cost = MSE(y, l2)\n",
    "\n",
    "feed_dict = {\n",
    "    X: X_,\n",
    "    y: y_,\n",
    "    W1: W1_,\n",
    "    b1: b1_,\n",
    "    W2: W2_,\n",
    "    b2: b2_\n",
    "}\n",
    "\n",
    "epochs = 1000\n",
    "# Total number of examples\n",
    "m = X_.shape[0]\n",
    "# 每次只取一个值:与后面对应\n",
    "batch_size = 1\n",
    "steps_per_epoch = m // batch_size\n",
    "\n",
    "graph = topological_sort_feed_dict(feed_dict)\n",
    "trainables = [W1, b1, W2, b2]\n",
    "\n",
    "print(\"Total number of examples = {}\".format(m))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, Loss: 35.672\n",
      "Epoch: 101, Loss: 14.352\n",
      "Epoch: 201, Loss: 12.876\n",
      "Epoch: 301, Loss: 17.916\n",
      "Epoch: 401, Loss: 19.008\n",
      "Epoch: 501, Loss: 15.787\n",
      "Epoch: 601, Loss: 22.702\n",
      "Epoch: 701, Loss: 18.774\n",
      "Epoch: 801, Loss: 18.395\n",
      "Epoch: 901, Loss: 18.804\n"
     ]
    }
   ],
   "source": [
    "losses = []\n",
    "\n",
    "for i in range(epochs):\n",
    "    loss = 0\n",
    "    for j in range(steps_per_epoch):\n",
    "        # Step 1\n",
    "        # Randomly sample a batch of examples\n",
    "        X_batch, y_batch = resample(X_, y_, n_samples=batch_size)\n",
    "\n",
    "        # Reset value of X and y Inputs\n",
    "        X.value = X_batch\n",
    "        y.value = y_batch\n",
    "\n",
    "        # Step 2\n",
    "    \n",
    "        forward_and_backward(graph) # set output node not important.\n",
    "\n",
    "        # Step 3\n",
    "        rate = 1e-2\n",
    "    \n",
    "        optimize(trainables, rate)\n",
    "\n",
    "        loss += graph[-1].value\n",
    "    \n",
    "    if i % 100 == 0: \n",
    "        print(\"Epoch: {}, Loss: {:.3f}\".format(i+1, loss/steps_per_epoch))\n",
    "        losses.append(loss/steps_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X in graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y in graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.Linear at 0x7fef15384d50>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph[-2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X.value = np.array([[6, 8]])\n",
    "# forward_and_backward(graph)\n",
    "# graph[-2].value[0][0]\n",
    "\n",
    "\n",
    "\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# 预测的y结果集\n",
    "predicate_result = []\n",
    "\n",
    "for rm, ls in X_.values:\n",
    "    X.value = np.array([[rm, ls]])\n",
    "    forward_and_backward(graph) # 得到此时预测y值\n",
    "    # 最终预测值存储在<__main__.Linear at 0x7f95b2ba1490>\n",
    "    predicate_result.append(graph[-2].value[0][0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[27.76771383549487,\n",
       " 24.002930326663915,\n",
       " 35.23738659905037,\n",
       " 35.62893550766687,\n",
       " 34.45209464356971,\n",
       " 26.32973863754321,\n",
       " 21.174144591556807,\n",
       " 17.072219931863977,\n",
       " 9.858225460774449,\n",
       " 17.32531045587139,\n",
       " 14.905540808435196,\n",
       " 20.685103781005296,\n",
       " 18.223908227600788,\n",
       " 21.727988944076884,\n",
       " 21.726581459697087,\n",
       " 21.591821073930724,\n",
       " 21.934717920620844,\n",
       " 19.214307818502185,\n",
       " 21.422864856842356,\n",
       " 21.439840096918797,\n",
       " 13.019536607878186,\n",
       " 20.23519673126239,\n",
       " 17.09963019188644,\n",
       " 15.211451518508007,\n",
       " 17.719140511096114,\n",
       " 17.143887287688045,\n",
       " 19.38387224721813,\n",
       " 17.26802984494377,\n",
       " 21.587712333799118,\n",
       " 24.09820433500419,\n",
       " 13.013312577984216,\n",
       " 20.8072733699509,\n",
       " 9.612925064666923,\n",
       " 15.587111962159588,\n",
       " 17.008220694192865,\n",
       " 21.594952783035335,\n",
       " 21.427058890726435,\n",
       " 21.587869643707158,\n",
       " 21.586252928616094,\n",
       " 29.752148454791,\n",
       " 39.57268857497692,\n",
       " 32.630864004360475,\n",
       " 23.59074408918124,\n",
       " 23.064476717665773,\n",
       " 21.783810010005055,\n",
       " 21.491702191081664,\n",
       " 20.190579096846214,\n",
       " 17.094271210784765,\n",
       " 11.576645377008841,\n",
       " 17.7029705237686,\n",
       " 20.588965760330304,\n",
       " 21.912492597655216,\n",
       " 26.832979666631033,\n",
       " 21.794851880928775,\n",
       " 19.249630223875613,\n",
       " 35.26963441131866,\n",
       " 25.68981646629554,\n",
       " 33.315855918128136,\n",
       " 22.83868943028918,\n",
       " 21.621385525284428,\n",
       " 20.99796262501293,\n",
       " 19.54176346086521,\n",
       " 25.796135184375274,\n",
       " 26.593487589741184,\n",
       " 33.14768035434234,\n",
       " 25.59099705307252,\n",
       " 21.50408521011392,\n",
       " 21.65055164815332,\n",
       " 21.11998816833085,\n",
       " 21.612509737617373,\n",
       " 25.465934997540288,\n",
       " 21.602684282700878,\n",
       " 22.906072428403814,\n",
       " 23.295654951415358,\n",
       " 24.00302957514113,\n",
       " 22.915352376387304,\n",
       " 21.575191589099482,\n",
       " 21.813121192625765,\n",
       " 21.28159217711341,\n",
       " 21.587196128136632,\n",
       " 31.121433866727532,\n",
       " 26.669411678336925,\n",
       " 24.333842017950087,\n",
       " 22.709945995169974,\n",
       " 23.39899856361305,\n",
       " 27.013254551318028,\n",
       " 20.959609043391758,\n",
       " 22.153642814878413,\n",
       " 33.32835383573898,\n",
       " 33.67746205471748,\n",
       " 24.17516891992488,\n",
       " 24.44477320382886,\n",
       " 24.845406755988417,\n",
       " 23.74462750450448,\n",
       " 22.057408435964778,\n",
       " 26.92431454120333,\n",
       " 21.611287783683935,\n",
       " 44.8971776839725,\n",
       " 47.07108366402167,\n",
       " 35.7420230846012,\n",
       " 26.448860966500213,\n",
       " 27.52030732032282,\n",
       " 22.928411965886745,\n",
       " 20.41154029722039,\n",
       " 21.235425879326762,\n",
       " 17.680496558889082,\n",
       " 16.834051199823794,\n",
       " 19.692703141303706,\n",
       " 22.112506270422145,\n",
       " 17.95558401909915,\n",
       " 20.78639246360697,\n",
       " 26.028731527468018,\n",
       " 17.788181640129995,\n",
       " 17.299000427154105,\n",
       " 22.12485419300147,\n",
       " 18.121610573117774,\n",
       " 21.381140704268333,\n",
       " 21.61901230263406,\n",
       " 18.603505494798927,\n",
       " 20.732846490975444,\n",
       " 19.803183080926825,\n",
       " 19.678335164464496,\n",
       " 17.170914399473453,\n",
       " 11.955638324797878,\n",
       " 17.232227215549532,\n",
       " 19.050278104102375,\n",
       " 12.664484393155682,\n",
       " 17.03126825742953,\n",
       " 18.067593676399955,\n",
       " 14.408923763615512,\n",
       " 21.655919863787908,\n",
       " 21.502380073246464,\n",
       " 22.414004870195665,\n",
       " 19.096620592585197,\n",
       " 17.2165617238442,\n",
       " 17.303537332821357,\n",
       " 17.415656349592734,\n",
       " 18.899732621485562,\n",
       " 13.880212304908838,\n",
       " 17.114182394067928,\n",
       " 12.088050743909033,\n",
       " 12.310721664363374,\n",
       " 13.000280393476563,\n",
       " 12.997360686839725,\n",
       " 13.004490759674425,\n",
       " 9.586079736111788,\n",
       " 17.20094310126334,\n",
       " 13.004455213578144,\n",
       " 13.00355742561591,\n",
       " 13.015212611996791,\n",
       " 19.68800384145628,\n",
       " 20.933897471443057,\n",
       " 17.59830116298984,\n",
       " 18.399170249884264,\n",
       " 18.455994095270203,\n",
       " 18.531232483480515,\n",
       " 14.84673793644289,\n",
       " 33.279192889455956,\n",
       " 22.506270152222633,\n",
       " 25.89835757562494,\n",
       " 24.59182547336481,\n",
       " 49.06597791803901,\n",
       " 51.10412620666014,\n",
       " 44.90000217465036,\n",
       " 21.39647868528928,\n",
       " 21.809793133937614,\n",
       " 45.4992932631793,\n",
       " 21.296445451503587,\n",
       " 22.14771651212847,\n",
       " 22.472907004096804,\n",
       " 19.723624234493833,\n",
       " 21.322080952092364,\n",
       " 19.898520536206462,\n",
       " 24.01729095308186,\n",
       " 21.55243422519293,\n",
       " 27.07167728528036,\n",
       " 21.63897156800505,\n",
       " 24.739050477726238,\n",
       " 30.905395043994226,\n",
       " 33.30778258446764,\n",
       " 38.53715722308729,\n",
       " 21.9953397127004,\n",
       " 34.71072708068297,\n",
       " 26.99019775989522,\n",
       " 20.595387814278222,\n",
       " 20.671148010156912,\n",
       " 45.6098467796197,\n",
       " 29.164910583629037,\n",
       " 28.115172988016894,\n",
       " 34.76399758706046,\n",
       " 33.22593675769859,\n",
       " 32.43586567061324,\n",
       " 37.22279389824374,\n",
       " 32.74466864647151,\n",
       " 29.851484375621748,\n",
       " 47.3301393133616,\n",
       " 35.8428133767393,\n",
       " 32.818067994045094,\n",
       " 35.01225171513813,\n",
       " 33.37385948686372,\n",
       " 34.68256274569691,\n",
       " 22.70988976982301,\n",
       " 46.85251954905041,\n",
       " 46.25797464398204,\n",
       " 45.46827183254738,\n",
       " 21.488623375831814,\n",
       " 22.242969296734454,\n",
       " 16.88767408285286,\n",
       " 19.09206620021328,\n",
       " 13.005748516732215,\n",
       " 17.29569698844938,\n",
       " 13.0048924481921,\n",
       " 18.047428465817227,\n",
       " 23.415677994588446,\n",
       " 12.64111124398868,\n",
       " 22.12754374731673,\n",
       " 20.62744787278502,\n",
       " 25.758002114600682,\n",
       " 17.172299910918504,\n",
       " 22.766982363722406,\n",
       " 27.4345387903766,\n",
       " 16.460358515479236,\n",
       " 26.9078865988663,\n",
       " 26.527622865353628,\n",
       " 44.876314076567695,\n",
       " 44.865157683906055,\n",
       " 45.215951213385864,\n",
       " 34.22451215667982,\n",
       " 47.05762060199851,\n",
       " 30.42771144568243,\n",
       " 21.404075199077294,\n",
       " 36.21997368393925,\n",
       " 45.19451531976808,\n",
       " 44.88021586679291,\n",
       " 26.924251470349766,\n",
       " 21.609763036880366,\n",
       " 25.75465820771744,\n",
       " 35.9855520512391,\n",
       " 26.161539738752857,\n",
       " 26.543962049852954,\n",
       " 26.12887813076074,\n",
       " 21.180438616621597,\n",
       " 22.278832130731132,\n",
       " 26.076809229111227,\n",
       " 21.294024293404334,\n",
       " 13.812449781883021,\n",
       " 21.945232443095726,\n",
       " 22.107922843397926,\n",
       " 23.87613124306969,\n",
       " 27.931211251772787,\n",
       " 26.398998511868392,\n",
       " 28.660978895759943,\n",
       " 33.90171269025279,\n",
       " 44.89444831146164,\n",
       " 22.70969354023746,\n",
       " 21.580744012883713,\n",
       " 42.196283205700865,\n",
       " 44.858037122479374,\n",
       " 34.88403855382482,\n",
       " 30.41623845417342,\n",
       " 32.6868737669491,\n",
       " 35.34858787845029,\n",
       " 44.85057274437558,\n",
       " 29.439154534607294,\n",
       " 33.936020443862446,\n",
       " 21.480536341389687,\n",
       " 19.049184152553885,\n",
       " 44.74347880881013,\n",
       " 42.57177339246297,\n",
       " 20.470187954360764,\n",
       " 20.988170184806986,\n",
       " 23.794577334978253,\n",
       " 25.949801387681575,\n",
       " 39.20921485398969,\n",
       " 33.79101000226417,\n",
       " 35.2613243847844,\n",
       " 35.08844115858783,\n",
       " 33.21423287583127,\n",
       " 25.774243649650536,\n",
       " 32.92059344145666,\n",
       " 46.786261433685496,\n",
       " 33.34483419073486,\n",
       " 47.56680008750213,\n",
       " 46.259422757077985,\n",
       " 33.13165220976184,\n",
       " 24.91131923618799,\n",
       " 20.847999577674194,\n",
       " 23.20090990163586,\n",
       " 23.91408305173303,\n",
       " 25.19625748570185,\n",
       " 34.1580486601659,\n",
       " 35.375782943254734,\n",
       " 29.741788547262676,\n",
       " 22.14175695431427,\n",
       " 21.596318373345046,\n",
       " 27.684655888088827,\n",
       " 26.183689287581416,\n",
       " 18.24467249906322,\n",
       " 25.83026378547136,\n",
       " 33.677116529103046,\n",
       " 32.5767813343567,\n",
       " 25.43146095640343,\n",
       " 25.050928100746702,\n",
       " 33.34210329999891,\n",
       " 34.696776813144055,\n",
       " 25.957816805843738,\n",
       " 35.65038208245245,\n",
       " 28.994996013020373,\n",
       " 30.354860617902975,\n",
       " 21.604709042906798,\n",
       " 17.405109924098074,\n",
       " 23.08939602384646,\n",
       " 21.399257264972462,\n",
       " 23.283279515167727,\n",
       " 25.35747203784507,\n",
       " 21.42138560794426,\n",
       " 17.10324294571434,\n",
       " 18.159711117128765,\n",
       " 22.911325974074735,\n",
       " 20.998918402618994,\n",
       " 25.273513064563904,\n",
       " 24.98839029711225,\n",
       " 22.03066413245297,\n",
       " 21.394907765941504,\n",
       " 25.7724975905641,\n",
       " 26.401191910468164,\n",
       " 24.797174421047195,\n",
       " 20.96998912988086,\n",
       " 21.540227585564836,\n",
       " 24.257358766023923,\n",
       " 22.079617470352083,\n",
       " 21.273686822309585,\n",
       " 21.972285590141873,\n",
       " 25.13121642375527,\n",
       " 24.39644250565861,\n",
       " 21.956010585945958,\n",
       " 21.549327667711268,\n",
       " 21.514591508135993,\n",
       " 21.927304381126227,\n",
       " 21.637347803290943,\n",
       " 21.65805190604237,\n",
       " 35.0785988579905,\n",
       " 25.483819696434963,\n",
       " 27.12793932080863,\n",
       " 33.15855249936985,\n",
       " 21.58619480538931,\n",
       " 21.11627325252266,\n",
       " 26.39457415656237,\n",
       " 27.428033015754142,\n",
       " 33.07453392107425,\n",
       " 26.38482213758323,\n",
       " 27.259769876216446,\n",
       " 21.681311339063193,\n",
       " 32.524177232177934,\n",
       " 21.537917971226335,\n",
       " 22.222083092689832,\n",
       " 17.190243891308,\n",
       " 20.62722362348125,\n",
       " 21.525960414760263,\n",
       " 21.035755104830898,\n",
       " 24.63468508553173,\n",
       " 19.407747246286828,\n",
       " 21.483340179985554,\n",
       " 19.609828193703144,\n",
       " 44.85125709951174,\n",
       " 35.085055315418884,\n",
       " 17.11720661873102,\n",
       " 17.657757668565246,\n",
       " 44.25105888465043,\n",
       " 33.18861209053043,\n",
       " 35.680679186491375,\n",
       " 22.2569328976299,\n",
       " 21.599597269138695,\n",
       " 12.812262870413205,\n",
       " 13.004480682197784,\n",
       " 21.601920730138854,\n",
       " 13.684873753441586,\n",
       " 13.949686129160137,\n",
       " 13.62360223336759,\n",
       " 15.381296886127005,\n",
       " 16.711675975570436,\n",
       " 13.73157772150982,\n",
       " 13.004826813858257,\n",
       " 13.003796205888557,\n",
       " 13.00451794046545,\n",
       " 12.751636975318181,\n",
       " 13.004530198351944,\n",
       " 12.992011998484838,\n",
       " 13.004357124597112,\n",
       " 13.018147994037417,\n",
       " 17.21135234402623,\n",
       " 17.098505104940998,\n",
       " 13.004638737418484,\n",
       " 18.3237791108549,\n",
       " 17.72273517496601,\n",
       " 17.225453535251315,\n",
       " 15.962380431233775,\n",
       " 13.976457023158066,\n",
       " 11.052685314657346,\n",
       " 9.583142790483222,\n",
       " 9.660498472659032,\n",
       " 15.591419558623516,\n",
       " 14.715456062476875,\n",
       " 13.052196989984653,\n",
       " 12.902784742877497,\n",
       " 13.00669596083817,\n",
       " 13.012862727628123,\n",
       " 21.358126396530185,\n",
       " 12.905917979309185,\n",
       " 14.440099101280945,\n",
       " 21.50375024958347,\n",
       " 13.765170803967145,\n",
       " 13.003794301384582,\n",
       " 13.051981932825559,\n",
       " 12.997724136215378,\n",
       " 9.889227646052227,\n",
       " 13.676416647555014,\n",
       " 13.004005781597467,\n",
       " 16.46825283457426,\n",
       " 13.841838380904836,\n",
       " 18.379854827799733,\n",
       " 18.06891441362186,\n",
       " 20.443298893792807,\n",
       " 13.458420991902434,\n",
       " 15.246982717011356,\n",
       " 12.510384036757118,\n",
       " 18.322893949840562,\n",
       " 19.04747714427559,\n",
       " 16.1285009992553,\n",
       " 13.595453890691054,\n",
       " 17.17095360143629,\n",
       " 14.378861199958795,\n",
       " 22.069697901978856,\n",
       " 17.577561597348137,\n",
       " 18.307506795873863,\n",
       " 13.677711270096792,\n",
       " 16.738828980331412,\n",
       " 9.67509413806911,\n",
       " 9.577755641389732,\n",
       " 13.006148274160571,\n",
       " 13.145542196084698,\n",
       " 15.751467256892113,\n",
       " 17.407523939029105,\n",
       " 15.531399245772656,\n",
       " 12.920753145876343,\n",
       " 13.635751028319113,\n",
       " 17.147873090063463,\n",
       " 17.45376059607342,\n",
       " 17.136107619677876,\n",
       " 15.875520054648002,\n",
       " 14.988276014958613,\n",
       " 14.999638002549949,\n",
       " 17.23855985434969,\n",
       " 19.58233568386084,\n",
       " 14.265825238721193,\n",
       " 15.998917596648106,\n",
       " 17.060718025406963,\n",
       " 17.404215429177235,\n",
       " 17.53148629413802,\n",
       " 19.0136812232913,\n",
       " 16.704553926613038,\n",
       " 18.7690995656037,\n",
       " 19.62511869521363,\n",
       " 24.14196582954905,\n",
       " 20.582092429562184,\n",
       " 20.253906607205888,\n",
       " 17.330506950506145,\n",
       " 16.313698220828087,\n",
       " 17.138468477008715,\n",
       " 19.635220694879962,\n",
       " 17.543481382302566,\n",
       " 20.89876305957934,\n",
       " 19.172580290928728,\n",
       " 25.927428022910508,\n",
       " 13.283644294998906,\n",
       " 12.013398808217035,\n",
       " 15.788533899937878,\n",
       " 13.004662369528601,\n",
       " 17.14477913060846,\n",
       " 20.684403060291473,\n",
       " 21.982001488979662,\n",
       " 27.20005624353894,\n",
       " 33.259074099730995,\n",
       " 21.492463012223517,\n",
       " 20.7713354015469,\n",
       " 22.34858554367955,\n",
       " 18.63064997131457,\n",
       " 21.426396289042692,\n",
       " 13.331560130022654,\n",
       " 13.004882648593288,\n",
       " 13.002899384148133,\n",
       " 17.152882566594556,\n",
       " 20.648178735011754,\n",
       " 21.356998396297467,\n",
       " 20.515087110552578,\n",
       " 16.272134610709678,\n",
       " 13.014398207119722,\n",
       " 20.228396126621195,\n",
       " 20.921157770897537,\n",
       " 19.33875684225641,\n",
       " 19.564858006430004,\n",
       " 25.353657052640212,\n",
       " 21.998768886159517,\n",
       " 33.21589341724635,\n",
       " 30.049437603898244,\n",
       " 21.95975197606257]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicate_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "/* Put everything inside the global mpl namespace */\n",
       "window.mpl = {};\n",
       "\n",
       "\n",
       "mpl.get_websocket_type = function() {\n",
       "    if (typeof(WebSocket) !== 'undefined') {\n",
       "        return WebSocket;\n",
       "    } else if (typeof(MozWebSocket) !== 'undefined') {\n",
       "        return MozWebSocket;\n",
       "    } else {\n",
       "        alert('Your browser does not have WebSocket support. ' +\n",
       "              'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
       "              'Firefox 4 and 5 are also supported but you ' +\n",
       "              'have to enable WebSockets in about:config.');\n",
       "    };\n",
       "}\n",
       "\n",
       "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n",
       "    this.id = figure_id;\n",
       "\n",
       "    this.ws = websocket;\n",
       "\n",
       "    this.supports_binary = (this.ws.binaryType != undefined);\n",
       "\n",
       "    if (!this.supports_binary) {\n",
       "        var warnings = document.getElementById(\"mpl-warnings\");\n",
       "        if (warnings) {\n",
       "            warnings.style.display = 'block';\n",
       "            warnings.textContent = (\n",
       "                \"This browser does not support binary websocket messages. \" +\n",
       "                    \"Performance may be slow.\");\n",
       "        }\n",
       "    }\n",
       "\n",
       "    this.imageObj = new Image();\n",
       "\n",
       "    this.context = undefined;\n",
       "    this.message = undefined;\n",
       "    this.canvas = undefined;\n",
       "    this.rubberband_canvas = undefined;\n",
       "    this.rubberband_context = undefined;\n",
       "    this.format_dropdown = undefined;\n",
       "\n",
       "    this.image_mode = 'full';\n",
       "\n",
       "    this.root = $('<div/>');\n",
       "    this._root_extra_style(this.root)\n",
       "    this.root.attr('style', 'display: inline-block');\n",
       "\n",
       "    $(parent_element).append(this.root);\n",
       "\n",
       "    this._init_header(this);\n",
       "    this._init_canvas(this);\n",
       "    this._init_toolbar(this);\n",
       "\n",
       "    var fig = this;\n",
       "\n",
       "    this.waiting = false;\n",
       "\n",
       "    this.ws.onopen =  function () {\n",
       "            fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n",
       "            fig.send_message(\"send_image_mode\", {});\n",
       "            if (mpl.ratio != 1) {\n",
       "                fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n",
       "            }\n",
       "            fig.send_message(\"refresh\", {});\n",
       "        }\n",
       "\n",
       "    this.imageObj.onload = function() {\n",
       "            if (fig.image_mode == 'full') {\n",
       "                // Full images could contain transparency (where diff images\n",
       "                // almost always do), so we need to clear the canvas so that\n",
       "                // there is no ghosting.\n",
       "                fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
       "            }\n",
       "            fig.context.drawImage(fig.imageObj, 0, 0);\n",
       "        };\n",
       "\n",
       "    this.imageObj.onunload = function() {\n",
       "        fig.ws.close();\n",
       "    }\n",
       "\n",
       "    this.ws.onmessage = this._make_on_message_function(this);\n",
       "\n",
       "    this.ondownload = ondownload;\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._init_header = function() {\n",
       "    var titlebar = $(\n",
       "        '<div class=\"ui-dialog-titlebar ui-widget-header ui-corner-all ' +\n",
       "        'ui-helper-clearfix\"/>');\n",
       "    var titletext = $(\n",
       "        '<div class=\"ui-dialog-title\" style=\"width: 100%; ' +\n",
       "        'text-align: center; padding: 3px;\"/>');\n",
       "    titlebar.append(titletext)\n",
       "    this.root.append(titlebar);\n",
       "    this.header = titletext[0];\n",
       "}\n",
       "\n",
       "\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n",
       "\n",
       "}\n",
       "\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n",
       "\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._init_canvas = function() {\n",
       "    var fig = this;\n",
       "\n",
       "    var canvas_div = $('<div/>');\n",
       "\n",
       "    canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n",
       "\n",
       "    function canvas_keyboard_event(event) {\n",
       "        return fig.key_event(event, event['data']);\n",
       "    }\n",
       "\n",
       "    canvas_div.keydown('key_press', canvas_keyboard_event);\n",
       "    canvas_div.keyup('key_release', canvas_keyboard_event);\n",
       "    this.canvas_div = canvas_div\n",
       "    this._canvas_extra_style(canvas_div)\n",
       "    this.root.append(canvas_div);\n",
       "\n",
       "    var canvas = $('<canvas/>');\n",
       "    canvas.addClass('mpl-canvas');\n",
       "    canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n",
       "\n",
       "    this.canvas = canvas[0];\n",
       "    this.context = canvas[0].getContext(\"2d\");\n",
       "\n",
       "    var backingStore = this.context.backingStorePixelRatio ||\n",
       "\tthis.context.webkitBackingStorePixelRatio ||\n",
       "\tthis.context.mozBackingStorePixelRatio ||\n",
       "\tthis.context.msBackingStorePixelRatio ||\n",
       "\tthis.context.oBackingStorePixelRatio ||\n",
       "\tthis.context.backingStorePixelRatio || 1;\n",
       "\n",
       "    mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
       "\n",
       "    var rubberband = $('<canvas/>');\n",
       "    rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n",
       "\n",
       "    var pass_mouse_events = true;\n",
       "\n",
       "    canvas_div.resizable({\n",
       "        start: function(event, ui) {\n",
       "            pass_mouse_events = false;\n",
       "        },\n",
       "        resize: function(event, ui) {\n",
       "            fig.request_resize(ui.size.width, ui.size.height);\n",
       "        },\n",
       "        stop: function(event, ui) {\n",
       "            pass_mouse_events = true;\n",
       "            fig.request_resize(ui.size.width, ui.size.height);\n",
       "        },\n",
       "    });\n",
       "\n",
       "    function mouse_event_fn(event) {\n",
       "        if (pass_mouse_events)\n",
       "            return fig.mouse_event(event, event['data']);\n",
       "    }\n",
       "\n",
       "    rubberband.mousedown('button_press', mouse_event_fn);\n",
       "    rubberband.mouseup('button_release', mouse_event_fn);\n",
       "    // Throttle sequential mouse events to 1 every 20ms.\n",
       "    rubberband.mousemove('motion_notify', mouse_event_fn);\n",
       "\n",
       "    rubberband.mouseenter('figure_enter', mouse_event_fn);\n",
       "    rubberband.mouseleave('figure_leave', mouse_event_fn);\n",
       "\n",
       "    canvas_div.on(\"wheel\", function (event) {\n",
       "        event = event.originalEvent;\n",
       "        event['data'] = 'scroll'\n",
       "        if (event.deltaY < 0) {\n",
       "            event.step = 1;\n",
       "        } else {\n",
       "            event.step = -1;\n",
       "        }\n",
       "        mouse_event_fn(event);\n",
       "    });\n",
       "\n",
       "    canvas_div.append(canvas);\n",
       "    canvas_div.append(rubberband);\n",
       "\n",
       "    this.rubberband = rubberband;\n",
       "    this.rubberband_canvas = rubberband[0];\n",
       "    this.rubberband_context = rubberband[0].getContext(\"2d\");\n",
       "    this.rubberband_context.strokeStyle = \"#000000\";\n",
       "\n",
       "    this._resize_canvas = function(width, height) {\n",
       "        // Keep the size of the canvas, canvas container, and rubber band\n",
       "        // canvas in synch.\n",
       "        canvas_div.css('width', width)\n",
       "        canvas_div.css('height', height)\n",
       "\n",
       "        canvas.attr('width', width * mpl.ratio);\n",
       "        canvas.attr('height', height * mpl.ratio);\n",
       "        canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n",
       "\n",
       "        rubberband.attr('width', width);\n",
       "        rubberband.attr('height', height);\n",
       "    }\n",
       "\n",
       "    // Set the figure to an initial 600x600px, this will subsequently be updated\n",
       "    // upon first draw.\n",
       "    this._resize_canvas(600, 600);\n",
       "\n",
       "    // Disable right mouse context menu.\n",
       "    $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n",
       "        return false;\n",
       "    });\n",
       "\n",
       "    function set_focus () {\n",
       "        canvas.focus();\n",
       "        canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    window.setTimeout(set_focus, 100);\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function() {\n",
       "    var fig = this;\n",
       "\n",
       "    var nav_element = $('<div/>');\n",
       "    nav_element.attr('style', 'width: 100%');\n",
       "    this.root.append(nav_element);\n",
       "\n",
       "    // Define a callback function for later on.\n",
       "    function toolbar_event(event) {\n",
       "        return fig.toolbar_button_onclick(event['data']);\n",
       "    }\n",
       "    function toolbar_mouse_event(event) {\n",
       "        return fig.toolbar_button_onmouseover(event['data']);\n",
       "    }\n",
       "\n",
       "    for(var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            // put a spacer in here.\n",
       "            continue;\n",
       "        }\n",
       "        var button = $('<button/>');\n",
       "        button.addClass('ui-button ui-widget ui-state-default ui-corner-all ' +\n",
       "                        'ui-button-icon-only');\n",
       "        button.attr('role', 'button');\n",
       "        button.attr('aria-disabled', 'false');\n",
       "        button.click(method_name, toolbar_event);\n",
       "        button.mouseover(tooltip, toolbar_mouse_event);\n",
       "\n",
       "        var icon_img = $('<span/>');\n",
       "        icon_img.addClass('ui-button-icon-primary ui-icon');\n",
       "        icon_img.addClass(image);\n",
       "        icon_img.addClass('ui-corner-all');\n",
       "\n",
       "        var tooltip_span = $('<span/>');\n",
       "        tooltip_span.addClass('ui-button-text');\n",
       "        tooltip_span.html(tooltip);\n",
       "\n",
       "        button.append(icon_img);\n",
       "        button.append(tooltip_span);\n",
       "\n",
       "        nav_element.append(button);\n",
       "    }\n",
       "\n",
       "    var fmt_picker_span = $('<span/>');\n",
       "\n",
       "    var fmt_picker = $('<select/>');\n",
       "    fmt_picker.addClass('mpl-toolbar-option ui-widget ui-widget-content');\n",
       "    fmt_picker_span.append(fmt_picker);\n",
       "    nav_element.append(fmt_picker_span);\n",
       "    this.format_dropdown = fmt_picker[0];\n",
       "\n",
       "    for (var ind in mpl.extensions) {\n",
       "        var fmt = mpl.extensions[ind];\n",
       "        var option = $(\n",
       "            '<option/>', {selected: fmt === mpl.default_extension}).html(fmt);\n",
       "        fmt_picker.append(option);\n",
       "    }\n",
       "\n",
       "    // Add hover states to the ui-buttons\n",
       "    $( \".ui-button\" ).hover(\n",
       "        function() { $(this).addClass(\"ui-state-hover\");},\n",
       "        function() { $(this).removeClass(\"ui-state-hover\");}\n",
       "    );\n",
       "\n",
       "    var status_bar = $('<span class=\"mpl-message\"/>');\n",
       "    nav_element.append(status_bar);\n",
       "    this.message = status_bar[0];\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.request_resize = function(x_pixels, y_pixels) {\n",
       "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
       "    // which will in turn request a refresh of the image.\n",
       "    this.send_message('resize', {'width': x_pixels, 'height': y_pixels});\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.send_message = function(type, properties) {\n",
       "    properties['type'] = type;\n",
       "    properties['figure_id'] = this.id;\n",
       "    this.ws.send(JSON.stringify(properties));\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.send_draw_message = function() {\n",
       "    if (!this.waiting) {\n",
       "        this.waiting = true;\n",
       "        this.ws.send(JSON.stringify({type: \"draw\", figure_id: this.id}));\n",
       "    }\n",
       "}\n",
       "\n",
       "\n",
       "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
       "    var format_dropdown = fig.format_dropdown;\n",
       "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
       "    fig.ondownload(fig, format);\n",
       "}\n",
       "\n",
       "\n",
       "mpl.figure.prototype.handle_resize = function(fig, msg) {\n",
       "    var size = msg['size'];\n",
       "    if (size[0] != fig.canvas.width || size[1] != fig.canvas.height) {\n",
       "        fig._resize_canvas(size[0], size[1]);\n",
       "        fig.send_message(\"refresh\", {});\n",
       "    };\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_rubberband = function(fig, msg) {\n",
       "    var x0 = msg['x0'] / mpl.ratio;\n",
       "    var y0 = (fig.canvas.height - msg['y0']) / mpl.ratio;\n",
       "    var x1 = msg['x1'] / mpl.ratio;\n",
       "    var y1 = (fig.canvas.height - msg['y1']) / mpl.ratio;\n",
       "    x0 = Math.floor(x0) + 0.5;\n",
       "    y0 = Math.floor(y0) + 0.5;\n",
       "    x1 = Math.floor(x1) + 0.5;\n",
       "    y1 = Math.floor(y1) + 0.5;\n",
       "    var min_x = Math.min(x0, x1);\n",
       "    var min_y = Math.min(y0, y1);\n",
       "    var width = Math.abs(x1 - x0);\n",
       "    var height = Math.abs(y1 - y0);\n",
       "\n",
       "    fig.rubberband_context.clearRect(\n",
       "        0, 0, fig.canvas.width / mpl.ratio, fig.canvas.height / mpl.ratio);\n",
       "\n",
       "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_figure_label = function(fig, msg) {\n",
       "    // Updates the figure title.\n",
       "    fig.header.textContent = msg['label'];\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_cursor = function(fig, msg) {\n",
       "    var cursor = msg['cursor'];\n",
       "    switch(cursor)\n",
       "    {\n",
       "    case 0:\n",
       "        cursor = 'pointer';\n",
       "        break;\n",
       "    case 1:\n",
       "        cursor = 'default';\n",
       "        break;\n",
       "    case 2:\n",
       "        cursor = 'crosshair';\n",
       "        break;\n",
       "    case 3:\n",
       "        cursor = 'move';\n",
       "        break;\n",
       "    }\n",
       "    fig.rubberband_canvas.style.cursor = cursor;\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_message = function(fig, msg) {\n",
       "    fig.message.textContent = msg['message'];\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_draw = function(fig, msg) {\n",
       "    // Request the server to send over a new figure.\n",
       "    fig.send_draw_message();\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_image_mode = function(fig, msg) {\n",
       "    fig.image_mode = msg['mode'];\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function() {\n",
       "    // Called whenever the canvas gets updated.\n",
       "    this.send_message(\"ack\", {});\n",
       "}\n",
       "\n",
       "// A function to construct a web socket function for onmessage handling.\n",
       "// Called in the figure constructor.\n",
       "mpl.figure.prototype._make_on_message_function = function(fig) {\n",
       "    return function socket_on_message(evt) {\n",
       "        if (evt.data instanceof Blob) {\n",
       "            /* FIXME: We get \"Resource interpreted as Image but\n",
       "             * transferred with MIME type text/plain:\" errors on\n",
       "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
       "             * to be part of the websocket stream */\n",
       "            evt.data.type = \"image/png\";\n",
       "\n",
       "            /* Free the memory for the previous frames */\n",
       "            if (fig.imageObj.src) {\n",
       "                (window.URL || window.webkitURL).revokeObjectURL(\n",
       "                    fig.imageObj.src);\n",
       "            }\n",
       "\n",
       "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
       "                evt.data);\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        }\n",
       "        else if (typeof evt.data === 'string' && evt.data.slice(0, 21) == \"data:image/png;base64\") {\n",
       "            fig.imageObj.src = evt.data;\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        var msg = JSON.parse(evt.data);\n",
       "        var msg_type = msg['type'];\n",
       "\n",
       "        // Call the  \"handle_{type}\" callback, which takes\n",
       "        // the figure and JSON message as its only arguments.\n",
       "        try {\n",
       "            var callback = fig[\"handle_\" + msg_type];\n",
       "        } catch (e) {\n",
       "            console.log(\"No handler for the '\" + msg_type + \"' message type: \", msg);\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        if (callback) {\n",
       "            try {\n",
       "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
       "                callback(fig, msg);\n",
       "            } catch (e) {\n",
       "                console.log(\"Exception inside the 'handler_\" + msg_type + \"' callback:\", e, e.stack, msg);\n",
       "            }\n",
       "        }\n",
       "    };\n",
       "}\n",
       "\n",
       "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
       "mpl.findpos = function(e) {\n",
       "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
       "    var targ;\n",
       "    if (!e)\n",
       "        e = window.event;\n",
       "    if (e.target)\n",
       "        targ = e.target;\n",
       "    else if (e.srcElement)\n",
       "        targ = e.srcElement;\n",
       "    if (targ.nodeType == 3) // defeat Safari bug\n",
       "        targ = targ.parentNode;\n",
       "\n",
       "    // jQuery normalizes the pageX and pageY\n",
       "    // pageX,Y are the mouse positions relative to the document\n",
       "    // offset() returns the position of the element relative to the document\n",
       "    var x = e.pageX - $(targ).offset().left;\n",
       "    var y = e.pageY - $(targ).offset().top;\n",
       "\n",
       "    return {\"x\": x, \"y\": y};\n",
       "};\n",
       "\n",
       "/*\n",
       " * return a copy of an object with only non-object keys\n",
       " * we need this to avoid circular references\n",
       " * http://stackoverflow.com/a/24161582/3208463\n",
       " */\n",
       "function simpleKeys (original) {\n",
       "  return Object.keys(original).reduce(function (obj, key) {\n",
       "    if (typeof original[key] !== 'object')\n",
       "        obj[key] = original[key]\n",
       "    return obj;\n",
       "  }, {});\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.mouse_event = function(event, name) {\n",
       "    var canvas_pos = mpl.findpos(event)\n",
       "\n",
       "    if (name === 'button_press')\n",
       "    {\n",
       "        this.canvas.focus();\n",
       "        this.canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    var x = canvas_pos.x * mpl.ratio;\n",
       "    var y = canvas_pos.y * mpl.ratio;\n",
       "\n",
       "    this.send_message(name, {x: x, y: y, button: event.button,\n",
       "                             step: event.step,\n",
       "                             guiEvent: simpleKeys(event)});\n",
       "\n",
       "    /* This prevents the web browser from automatically changing to\n",
       "     * the text insertion cursor when the button is pressed.  We want\n",
       "     * to control all of the cursor setting manually through the\n",
       "     * 'cursor' event from matplotlib */\n",
       "    event.preventDefault();\n",
       "    return false;\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
       "    // Handle any extra behaviour associated with a key event\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.key_event = function(event, name) {\n",
       "\n",
       "    // Prevent repeat events\n",
       "    if (name == 'key_press')\n",
       "    {\n",
       "        if (event.which === this._key)\n",
       "            return;\n",
       "        else\n",
       "            this._key = event.which;\n",
       "    }\n",
       "    if (name == 'key_release')\n",
       "        this._key = null;\n",
       "\n",
       "    var value = '';\n",
       "    if (event.ctrlKey && event.which != 17)\n",
       "        value += \"ctrl+\";\n",
       "    if (event.altKey && event.which != 18)\n",
       "        value += \"alt+\";\n",
       "    if (event.shiftKey && event.which != 16)\n",
       "        value += \"shift+\";\n",
       "\n",
       "    value += 'k';\n",
       "    value += event.which.toString();\n",
       "\n",
       "    this._key_event_extra(event, name);\n",
       "\n",
       "    this.send_message(name, {key: value,\n",
       "                             guiEvent: simpleKeys(event)});\n",
       "    return false;\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onclick = function(name) {\n",
       "    if (name == 'download') {\n",
       "        this.handle_save(this, null);\n",
       "    } else {\n",
       "        this.send_message(\"toolbar_button\", {name: name});\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onmouseover = function(tooltip) {\n",
       "    this.message.textContent = tooltip;\n",
       "};\n",
       "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
       "\n",
       "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
       "\n",
       "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n",
       "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
       "    // object with the appropriate methods. Currently this is a non binary\n",
       "    // socket, so there is still some room for performance tuning.\n",
       "    var ws = {};\n",
       "\n",
       "    ws.close = function() {\n",
       "        comm.close()\n",
       "    };\n",
       "    ws.send = function(m) {\n",
       "        //console.log('sending', m);\n",
       "        comm.send(m);\n",
       "    };\n",
       "    // Register the callback with on_msg.\n",
       "    comm.on_msg(function(msg) {\n",
       "        //console.log('receiving', msg['content']['data'], msg);\n",
       "        // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
       "        ws.onmessage(msg['content']['data'])\n",
       "    });\n",
       "    return ws;\n",
       "}\n",
       "\n",
       "mpl.mpl_figure_comm = function(comm, msg) {\n",
       "    // This is the function which gets called when the mpl process\n",
       "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
       "\n",
       "    var id = msg.content.data.id;\n",
       "    // Get hold of the div created by the display call when the Comm\n",
       "    // socket was opened in Python.\n",
       "    var element = $(\"#\" + id);\n",
       "    var ws_proxy = comm_websocket_adapter(comm)\n",
       "\n",
       "    function ondownload(figure, format) {\n",
       "        window.open(figure.imageObj.src);\n",
       "    }\n",
       "\n",
       "    var fig = new mpl.figure(id, ws_proxy,\n",
       "                           ondownload,\n",
       "                           element.get(0));\n",
       "\n",
       "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
       "    // web socket which is closed, not our websocket->open comm proxy.\n",
       "    ws_proxy.onopen();\n",
       "\n",
       "    fig.parent_element = element.get(0);\n",
       "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
       "    if (!fig.cell_info) {\n",
       "        console.error(\"Failed to find cell for figure\", id, fig);\n",
       "        return;\n",
       "    }\n",
       "\n",
       "    var output_index = fig.cell_info[2]\n",
       "    var cell = fig.cell_info[0];\n",
       "\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_close = function(fig, msg) {\n",
       "    var width = fig.canvas.width/mpl.ratio\n",
       "    fig.root.unbind('remove')\n",
       "\n",
       "    // Update the output cell to use the data from the current canvas.\n",
       "    fig.push_to_output();\n",
       "    var dataURL = fig.canvas.toDataURL();\n",
       "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
       "    // the notebook keyboard shortcuts fail.\n",
       "    IPython.keyboard_manager.enable()\n",
       "    $(fig.parent_element).html('<img src=\"' + dataURL + '\" width=\"' + width + '\">');\n",
       "    fig.close_ws(fig, msg);\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.close_ws = function(fig, msg){\n",
       "    fig.send_message('closing', msg);\n",
       "    // fig.ws.close()\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.push_to_output = function(remove_interactive) {\n",
       "    // Turn the data on the canvas into data in the output cell.\n",
       "    var width = this.canvas.width/mpl.ratio\n",
       "    var dataURL = this.canvas.toDataURL();\n",
       "    this.cell_info[1]['text/html'] = '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function() {\n",
       "    // Tell IPython that the notebook contents must change.\n",
       "    IPython.notebook.set_dirty(true);\n",
       "    this.send_message(\"ack\", {});\n",
       "    var fig = this;\n",
       "    // Wait a second, then push the new image to the DOM so\n",
       "    // that it is saved nicely (might be nice to debounce this).\n",
       "    setTimeout(function () { fig.push_to_output() }, 1000);\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function() {\n",
       "    var fig = this;\n",
       "\n",
       "    var nav_element = $('<div/>');\n",
       "    nav_element.attr('style', 'width: 100%');\n",
       "    this.root.append(nav_element);\n",
       "\n",
       "    // Define a callback function for later on.\n",
       "    function toolbar_event(event) {\n",
       "        return fig.toolbar_button_onclick(event['data']);\n",
       "    }\n",
       "    function toolbar_mouse_event(event) {\n",
       "        return fig.toolbar_button_onmouseover(event['data']);\n",
       "    }\n",
       "\n",
       "    for(var toolbar_ind in mpl.toolbar_items){\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) { continue; };\n",
       "\n",
       "        var button = $('<button class=\"btn btn-default\" href=\"#\" title=\"' + name + '\"><i class=\"fa ' + image + ' fa-lg\"></i></button>');\n",
       "        button.click(method_name, toolbar_event);\n",
       "        button.mouseover(tooltip, toolbar_mouse_event);\n",
       "        nav_element.append(button);\n",
       "    }\n",
       "\n",
       "    // Add the status bar.\n",
       "    var status_bar = $('<span class=\"mpl-message\" style=\"text-align:right; float: right;\"/>');\n",
       "    nav_element.append(status_bar);\n",
       "    this.message = status_bar[0];\n",
       "\n",
       "    // Add the close button to the window.\n",
       "    var buttongrp = $('<div class=\"btn-group inline pull-right\"></div>');\n",
       "    var button = $('<button class=\"btn btn-mini btn-primary\" href=\"#\" title=\"Stop Interaction\"><i class=\"fa fa-power-off icon-remove icon-large\"></i></button>');\n",
       "    button.click(function (evt) { fig.handle_close(fig, {}); } );\n",
       "    button.mouseover('Stop Interaction', toolbar_mouse_event);\n",
       "    buttongrp.append(button);\n",
       "    var titlebar = this.root.find($('.ui-dialog-titlebar'));\n",
       "    titlebar.prepend(buttongrp);\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function(el){\n",
       "    var fig = this\n",
       "    el.on(\"remove\", function(){\n",
       "\tfig.close_ws(fig, {});\n",
       "    });\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function(el){\n",
       "    // this is important to make the div 'focusable\n",
       "    el.attr('tabindex', 0)\n",
       "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
       "    // off when our div gets focus\n",
       "\n",
       "    // location in version 3\n",
       "    if (IPython.notebook.keyboard_manager) {\n",
       "        IPython.notebook.keyboard_manager.register_events(el);\n",
       "    }\n",
       "    else {\n",
       "        // location in version 2\n",
       "        IPython.keyboard_manager.register_events(el);\n",
       "    }\n",
       "\n",
       "}\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
       "    var manager = IPython.notebook.keyboard_manager;\n",
       "    if (!manager)\n",
       "        manager = IPython.keyboard_manager;\n",
       "\n",
       "    // Check for shift+enter\n",
       "    if (event.shiftKey && event.which == 13) {\n",
       "        this.canvas_div.blur();\n",
       "        // select the cell after this one\n",
       "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
       "        IPython.notebook.select(index + 1);\n",
       "    }\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
       "    fig.ondownload(fig, null);\n",
       "}\n",
       "\n",
       "\n",
       "mpl.find_output_cell = function(html_output) {\n",
       "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
       "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
       "    // IPython event is triggered only after the cells have been serialised, which for\n",
       "    // our purposes (turning an active figure into a static one), is too late.\n",
       "    var cells = IPython.notebook.get_cells();\n",
       "    var ncells = cells.length;\n",
       "    for (var i=0; i<ncells; i++) {\n",
       "        var cell = cells[i];\n",
       "        if (cell.cell_type === 'code'){\n",
       "            for (var j=0; j<cell.output_area.outputs.length; j++) {\n",
       "                var data = cell.output_area.outputs[j];\n",
       "                if (data.data) {\n",
       "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
       "                    data = data.data;\n",
       "                }\n",
       "                if (data['text/html'] == html_output) {\n",
       "                    return [cell, data, j];\n",
       "                }\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "}\n",
       "\n",
       "// Register the function which deals with the matplotlib target/channel.\n",
       "// The kernel may be null if the page has been refreshed.\n",
       "if (IPython.notebook.kernel != null) {\n",
       "    IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n",
       "}\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<img src=\"\" width=\"999.1666666666667\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Predicated-Price')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 画出三维图像\n",
    "# 图像可转动 \n",
    "%matplotlib notebook \n",
    "\n",
    "fig = plt.figure(figsize=(10, 10))\n",
    "ax = fig.add_subplot(111, projection='3d')\n",
    "\n",
    "# Make data.\n",
    "X = X_.values[:, 0]\n",
    "Y = X_.values[:, 1] # 二维\n",
    "Z = predicate_result\n",
    "\n",
    "# Plot the surface.\n",
    "rm_and_lstp_price = ax.plot_trisurf(X, Y, Z, color='Green')\n",
    "\n",
    "ax.set_xlabel('RM')\n",
    "ax.set_ylabel('% of lower state')\n",
    "ax.set_zlabel('Predicated-Price')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
